o
    1XxiU                   
   @   s0  d dl Z d dlmZmZmZ d dlZd dlZd dlmZm	Z	m
Z
mZmZmZmZ d dlZd dlmZ d dlm  m  mZ d dlmZ d dlmZ d dlmZ deeef fd	d
ZdededefddZdededededdf
ddZdededededdf
ddZ deddfddZ!deddfddZ"dS )    N)FutureThreadPoolExecutorwait)AnyDictListOptionalSetTuplecast)	ClientAPI)	RecordSet)test_hnsw_config)Metadatareturnc                  C   s    t dd} t dd}| |fS )N
   i'     )randomrandint)ND r   _/home/air/biblejyuku/back/venv/lib/python3.10/site-packages/chromadb/test/test_multithreaded.pygenerate_data_shape   s   r   r   r   c                 C   sZ   dd t | D }dd t | D }dd t | D }tj| | }||||d}|S )Nc                 S   s   g | ]}t |qS r   )str.0ir   r   r   
<listcomp>       z'generate_record_set.<locals>.<listcomp>c                 S   s   g | ]}| |iqS r   r   r   r   r   r   r          c                 S   s   g | ]}d | qS )zdoc r   r   r   r   r   r      r    ids
embeddings	metadatas	documents)rangenpr   randtolist)r   r   r"   r$   r%   r#   normalized_record_setr   r   r   generate_record_set   s   r+   clientnum_workersc              	   C   s  t ||}|d }|d }|d }|d }td| d| d| d |   | jd	td
}	t|dx}
g }d}|t|k rtd|d }t	|t|| }|d }|| d }|d urht||| dkrhn<|
j
|	j||| |d ur{||| nd |d ur||| nd |d ur||| nd d}|| ||7 }|t|k s>W d    n1 sw   Y  t| |D ]}| }|d ur|qt|	| t|	| t|	| t|	 tdd t|D d}d}tj|	|||d d S )Nr"   r#   r$   r%   Adding  records with  dimensions on  workerstestnamemetadatamax_workers   r   r   r!   c                 S      g | ]}|qS r   r   r   r   r   r   r   Z       z+_test_multithreaded_add.<locals>.<listcomp>   	n_resultsquery_indices)r+   printresetcreate_collectionr   r   lenr   r   minsubmitaddappendr   	exception
invariantscount	ids_matchmetadatas_matchno_duplicatessampler&   ann_accuracy)r,   r   r   r-   records_setr"   r#   r$   r%   collexecutorfutures
total_sent
batch_sizeto_sendstartendfuturerH   r?   r>   r   r   r   _test_multithreaded_add(   s`   




rZ   c              	      s&  |    | jdtdt||}ttt |d ttt |d ttt |d |d t	  t
 td| d| d	| d
 	ddtdttt  ddf fdd}t|de}g }d}|td k rtdd}	|	dkrtd|d }
t|
t| }|d }|| d }|||	|| }|| ||7 }n|	dkr|||	}|| |td k soW d   n1 sw   Y  t| |D ]}| }|dur|qt| t| t| t tdd t|D d}d}tj|||d dS )zeTest that will use multiple threads to interleave operations on the db and verify they work correctlyr2   r3   r"   r#   r$   r%   r.   r/   r0   r1   N	operationids_to_modifyr   c                    sv  | dkre|dus
J fdd|D }t |dkrdS j|dur+fdd|D nddur9fdd|D nddurGfdd|D ndd   | W d   dS 1 s^w   Y  dS | d	krg }d
} t  }W d   n1 sw   Y  fdd|D }t |dkst ||k rdS t|tdt |}fdd|D }j||d dS dS )z,Perform a random operation on the collectionr   Nc                       g | ]}  |qS r   indexr   idr"   r   r   r   }   r    zJ_test_interleaved_add_query.<locals>.perform_operation.<locals>.<listcomp>c                       g | ]} | qS r   r   r   r#   r   r   r      r   c                    rc   r   r   r   )r$   r   r   r      r   c                    rc   r   r   r   )r%   r   r   r      r   r!   r9   r<   c                    r]   r   r^   r`   rb   r   r   r      r    r   c                    rc   r   r   r   rd   r   r   r      r   )r>   )	rC   rF   updatelistcopyr   rN   rD   query)r[   r\   indices_to_modifycurrently_added_idsr>   currently_added_indicesr?   query_vectors	added_idsrQ   r%   r#   r"   lockr$   r   r   perform_operationw   sP   "
z6_test_interleaved_add_query.<locals>.perform_operationr6   r8   r9   r      r   c                 S   r:   r   r   r   r   r   r   r      r;   z/_test_interleaved_add_query.<locals>.<listcomp>r<   r=   )N) rA   rB   r   r+   r   r   r   floatr   set	threadingLockr@   intr   r   rC   r   r   rD   rE   rG   r   rH   rI   rJ   rK   rL   rM   rN   r&   rO   )r,   r   r   r-   rP   rp   rR   rS   rT   r[   rU   rV   rW   rX   rY   rH   r?   r>   r   rm   r   _test_interleaved_add_queryd   st   

2




rw   c                 C   >   t dD ]}tdt d }t \}}t| ||| qd S N   rq   )r&   r   r   multiprocessing	cpu_countr   rZ   r,   r   r-   r   r   r   r   r   test_multithreaded_add   
   
r~   c                 C   rx   ry   )r&   r   r   r{   r|   r   rw   r}   r   r   r   test_interleaved_add_query   r   r   )#r{   concurrent.futuresr   r   r   r   rt   typingr   r   r   r   r	   r
   r   numpyr'   chromadb.apir   !chromadb.test.property.invariantsr2   propertyrI   !chromadb.test.property.strategiesr   r   chromadb.typesr   rv   r   r+   rZ   rw   r~   r   r   r   r   r   <module>   sJ    $
<
r