o
    3Ih\                  
   @   s  d dl Z d dlZd dlmZ d dlZd dlZd dlmZmZm	Z	m
Z
mZ d dlmZ d dlmZ d dlmZ d dlZd dlZd dlmZmZ d dlmZmZ d dlmZ d d	lmZ d dlm   m!  mZ d dl"m   m!  m#Z# d d
lm$Z$ d dl%m&Z&m'Z'm(Z( d dl)m*Z*m+Z+m,Z,m-Z-m.Z. d dl/Z/d dl0m1Z2 d dl3m4Z4 d dl5Z6d dl7Z7eg ef Z8de/j9v reddddddde7: dgneddddddde7: dgZ;ej<de;ddej=deeddf fddZ>ej?ej@deAddddddZBejCd ejDde
ejEejFf fd!d"ZGeejHeG d#d$d% d$d% fd&d'ded(e	e
ejEejFf  ddfd)d*ZIdeddfd+d,ZJded-eKd.ejFd/eddf
d0d1ZLd2d3 ZMG d4d5 d5e&ZNd6ZOG d7d8 d8e(ZPd9ejQdeddfd:d;ZRd9ejQdeddfd<d=ZSdeddfd>d?ZTd9ejQdeddfd@dAZUdS )B    N)
Connection)	GeneratorCallableListTuplecast)UUID)given)	ClientAPI	ServerAPI)SettingsSystem)VectorReader)LocalSegmentManager)hashing_embedding_function)EmbeddingStateMachineStatestraceEmbeddingStateMachineBase)run_state_machine_as_testruleprecondition
initializeMultipleResults)Client)DefaultEmbeddingFunctionCHROMA_RUST_BINDINGS_TEST_ONLYz!chromadb.api.rust.RustBindingsAPIz chromadb.db.impl.sqlite.SqliteDBz7chromadb.segment.impl.manager.local.LocalSegmentManagerT)chroma_api_implchroma_sysdb_implchroma_producer_implchroma_consumer_implchroma_segment_manager_implallow_resetis_persistentpersist_directoryzchromadb.api.segment.SegmentAPImodule)scopeparamsrequestreturnc                 c   s    | j V  d S N)param)r'    r+   c/home/air/sanwanet/gpt-api/venv/lib/python3.10/site-packages/chromadb/test/property/test_persist.pysettingsD   s   r-   
   with_hnsw_paramswith_persistent_hnsw_paramsmax_hnsw_batch_sizemax_hnsw_sync_thresholdcoll)keydrawc                 C   s8   | t jdtdddd}| t t|}||fS )NTr.   r/   )
strategiescollectionsstjust
recordsets)r6   
collection	recordsetr+   r+   r,   !collection_and_recordset_strategyV   s   	r>      c                 C   s
   | d j S )Nr   )name)xr+   r+   r,   <lambda>k   s   
 rB   )min_size	unique_by)#collection_and_recordset_strategiesrE   c           	      C   s0  t | }|  t|}|  |D ]8\}}|j|j|j|jd}|j	di | t
|| t
|| t
|| t
|| t
j|||jd q|  ~~t | }|  t|}|D ].\}}|j|j|jd}t
|| t
|| t
|| t
|| t
j|||jd qa|  ~~d S )Nr@   metadataembedding_function)rH   r@   rH   r+   )r   startClientCreatorfrom_systemresetcreate_collectionr@   rG   rH   add
invariantscountmetadatas_matchdocuments_match	ids_matchann_accuracystopget_collection)	r-   rE   system_1client_1collection_strategyrecordset_strategyr4   system_2client_2r+   r+   r,   test_persistg   sb   

r^   c                    sn  t | }|  t|}|jddddd}|t}||jt	 dt
f fdd}| }|jdd	gd
gdggd | |ksCJ | }|jdgdggd | |ksWJ | }|jg dd
gdgdggd | |kspJ | }|jdgdggd |jdd	gd
gdggd | |ksJ | }|jdgdggd |jdd	gd
gdggd | |ksJ | }d S )Ntest   )hnsw:batch_sizehnsw:sync_threshold)r@   rG   r(   c                      s2   t d z	tj  W S  ty   Y dS w )Ng?)timesleepospathgetmtime_get_metadata_fileFileNotFoundErrorr+   segmentr+   r,   get_index_last_modified_at   s   
z7test_sync_threshold.<locals>.get_index_last_modified_at12      ?       @)ids
embeddings3      @)rn   ro   rt   4      @5      @)r   rJ   rK   rL   rN   instancer   get_segmentidr   floatrO   upsert)r-   systemclientr<   managerrm   last_modified_atr+   rk   r,   test_sync_threshold   s6   



r   collection_name
record_setconnc              
   C   s   z;t | }|  t|}|j|t d}t|| t	|| t
|| t|| t|| |  W d S  tyN } z|| |d }~ww )NrI   )r   rJ   rK   rL   rW   r7   "not_implemented_embedding_functionrP   rQ   rR   rS   rT   rU   rV   	Exceptionsend)r-   r   r   r   r   r   r4   er+   r+   r,   load_and_check   s&   

r   c                  C   s:   zt d} | dg | W S  ty   t d Y S w )N
forkserverchromadbspawn)multiprocessingget_contextset_forkserver_preloadr   )ctxr+   r+   r,   get_multiprocessing_context  s   
r   c                   @   s   e Zd ZdZdS )#PersistEmbeddingsStateMachineStatespersistN)__name__
__module____qualname__r   r+   r+   r+   r,   r     s    r      c                       s|   e Zd Zdedef fddZeeddej	fddZe
d	d
 e dddZdeddf fddZdddZ  ZS )PersistEmbeddingsStateMachiner   r-   c                    s.   || _ || _t| _| j   t | j  d S r)   )r   r-    MIN_STATE_CHANGES_BEFORE_PERSIST(min_state_changes_left_before_persistingrM   super__init__)selfr   r-   	__class__r+   r,   r     s
   
z&PersistEmbeddingsStateMachine.__init__r<   r<   c                 C   sX   | j   | j j|j|j|jd| _|j| _td | t	j
 tjg g g g d| _d S )NrF   init)rr   	metadatas	documentsrs   )r   rM   rN   r@   rG   rH   r<   r   on_state_changer   r   r7   StateMachineRecordSetrecord_set_state)r   r<   r+   r+   r,   r   !  s   
z(PersistEmbeddingsStateMachine.initializec                 C   s   t | jd dko| jdkS )Nrr   r?   r   )lenr   r   r   r+   r+   r,   rB   2  s    
z&PersistEmbeddingsStateMachine.<lambda>r(   Nc                 C   sp   |  tj | jj}t \}}t }|jt	| j
|| j|fd}|  |  | r2| }||  d S )N)targetargs)r   r   r   r<   r@   r   Piper   Processr   r-   r   rJ   joinpollrecvclose)r   r   conn1conn2r   pr   r+   r+   r,   r   1  s   z%PersistEmbeddingsStateMachine.persist	new_statec                    s2   t  | |tjkrt| _d S |  jd8  _d S )Nr?   )r   r   r   r   r   r   )r   r   r   r+   r,   r   H  s
   
z-PersistEmbeddingsStateMachine.on_state_changec                 C   s   | j   d S r)   )r   rM   r   r+   r+   r,   teardownQ  s   z&PersistEmbeddingsStateMachine.teardown)r(   N)r   r   r   r
   r   r   r   collection_str7   
Collectionr   r   r   strr   r   __classcell__r+   r+   r   r,   r     s    	r   caplogc                    s,   |  tj t t fdd d S )Nc                      s   t  dS )Nr-   r   )r   r+   r   r-   r+   r,   rB   [  s    z/test_persist_embeddings_state.<locals>.<lambda>)	set_levelloggingERRORr   r   r   )r   r-   r+   r   r,   test_persist_embeddings_stateU  s
   
r   c                 C   s8  t |}t||d}|jtjdddddddd tddtji g dd	d

d |	  |
  |  |  |  |jdgddggd gd dd\}|	  |
  |  |  |  |jddgddgddggd d gd dd\}}|	  |
  |  |  |  |j|gd |	  |  d S )Nr   A00   r`   hnsw:construction_efhnsw:search_efhnsw:Mrb   ra   z$2d3eddc7-2314-45f4-a951-47a9a8e099d2   FT
r@   rG   rH   r|   	dimensiondtypeknown_metadata_keysknown_document_keywordshas_documentshas_embeddingsr   0g      ?g     ?rr   rs   r   r   r   rn   ro   g     P?g     ?g     ÿg     ?rr   )r   r   r   r   r7   r   r   npfloat16rU   rQ   fields_matchlog_size_below_maxno_duplicatesadd_embeddingsdelete_by_idsr   )r   r-   r   stateembedding_ids_0embedding_ids_1embedding_ids_2r+   r+   r,   test_delete_less_than_k_  sR   
".r   c                 C   s   t | }t| |d}|jtjdddddddt tddd i g dd	d

d |jg dddgddgddggg dd dd |j	dgd |jdgddggd gd dd |
  d S )Nr   r   r   r`   )r   r   r   ra   rb   z$0851f751-2f11-4424-ab23-4ae97074887ar   FTr   r   )r   rn   ro   r   )NNNr   r   r   r   r?   )r   r   r   r   r7   r   r   r   r   r   r   )r-   r   r   r+   r+   r,   test_delete_add_after_persist  sJ   

r   c                 C   sn  t |}t||d}|jtjdddddddtdtjdt	d	dtji g d
dd
d |
  |  |  |  |  ttt |jg dg dg dg dg dg dg dg dgd d dd\}}}}}}	}
|
  |  |  |  |  td |jdgg dgd d dd}|
  |  |  |  |  ttt |jg dg dg dg dg dg d g d!g d"g d#g d$g d%g d&g d'g d(g d)g d*g d+g d,gd d dd\}}}}}}}}}}}}}}}}}|
  |  |  |  |  |jd-d.gg d/g d0gd d gd dd |
  |  |  |  |  |jg d1g d2g d3g d4g d5g d6g d7gd d dd |
  |  |  |  |  |  |
  |  |  |  |  |jd8g d9gd d dd |
  |  |  |  |  |jd:d;gg d<g d=gd d gd dd |
  |  |  |  |  |jd>d?d@dA|dBdCdDdE||dFdGdH|dIdJgg dg dg dg dg d g d!g d"g d#g d$g d%g d&g d'g d(g d)g d*g d+g d,gd d dd |
  |  |  |  |  d S )KNr   JqzMs4pPm14cr   	      r   \   )dimr   z$45c5c816-0a90-4293-8d01-4325ff860040FTr   r   )Ne8r6rv   YaoqFjA2cjHCvro   )        r   r   )rp   rp   rp   )rq   rq   rq   )ru   ru   ru   )rw   rw   rw   )ry   ry   ry   )      @r   r   r   r   z

	MVu393QTc)      @r   r   )CyF0Mk-q_Fwu2D2sQSFogDgPLkcfTSrwuQHQ6w4f51qWr2enLPQw8uKYs1Gwdzt5W8tpsnfJbV7zrx   V1iFkoJXZw4uFc7vEEwrPYf)       @r   r   )      "@r  r  )      $@r  r  )      &@r  r  )      (@r  r  )      *@r  r  )      ,@r  r  )      .@r  r  )      0@r  r  )      1@r	  r	  )      2@r
  r
  )      3@r  r  )      4@r  r  )      5@r  r  )      6@r  r  )      7@r  r  )      8@r  r  r   df_RWhR0HelOcv)      9@r  r  )      :@r  r  )3R9_44u3BMZCXZDSUelx)      ;@r  r  )      <@r  r  )      =@r  r  )      >@r  r  )      ?@r  r  )      @@r  r  YlVm)     @@r!  r!  Rk1TPL)      A@r$  r$  )     A@r%  r%  r   r   r   r   r   r   r   r   r   r   r   r   r   )r   r   r   r   r7   r   r   r   float64r   rU   rQ   r   r   r   r   r   r   r   printr   r   )r   r-   r   r   r   r   r   embedding_ids_3embedding_ids_4embedding_ids_5embedding_ids_6_embedding_ids_12embedding_ids_17embedding_ids_18embedding_ids_22r+   r+   r,   Mtest_batch_size_less_than_sync_with_duplicate_adds_results_in_skipped_seq_ids  s  
		0,r1  )Vr   r   multiprocessing.connectionr   multiprocessing.contextrd   typingr   r   r   r   r   uuidr   
hypothesisr	   hypothesis.strategiesr7   r9   pytestr   chromadb.apir
   r   chromadb.configr   r   chromadb.segmentr   #chromadb.segment.impl.manager.localr   !chromadb.test.property.strategiesr_   property!chromadb.test.property.invariantsrP   r   &chromadb.test.property.test_embeddingsr   r   r   hypothesis.statefulr   r   r   r   r   rf   chromadb.api.clientr   rK   "chromadb.utils.embedding_functionsr   numpyr   tempfileCreatePersistAPIenvironmkdtempconfigurationsfixtureFixtureRequestr-   sharedr8   r:   r   	compositeDrawFnr   	RecordSetr>   listsr^   r   r   r   r   r   r   r   LogCaptureFixturer   r   r   r1  r+   r+   r+   r,   <module>   s    
 @;
<


23