o
    1Xxiu                     @   sL  U d dl Zd dlZd dlmZ d dlZd dlZd dlZd dlmZ d dlmZm	Z	m
Z
 d dlmZmZmZmZmZmZmZ d dlmZ d dlmZmZmZmZmZ d dlmZ d dlmZ d dlm Z  d dl!m"  m#  mZ d d	l m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z- d d
l.m/Z/ d dl0m"  m#  m1Z1 d dl2m3Z3m4Z4 d dl5Z6d dl7Z7d dl8m9Z9m:Z: e/dd a;ee<e=f e>d< de<ddfddZ?dXddZ@ejAeBejCddZDejEee6jFe6jGe6jHf  e>d< ejAejIdddddZJejEe= e>d< eG dd dZKejAej.d d!d"dZLG d#d$ d$e%ZMG d%d& d&eMZNd'ejOd(eddfd)d*ZPd(eddfd+d,ZQd'ejOd(eddfd-d.ZRd(eddfd/d0ZSd(eddfd1d2ZTd(eddfd3d4ZUd(eddfd5d6ZVd(eddfd7d8ZWejXjYd9d:d(eddfd;d<ZZejX[d=i d>g id?i id@i ii i dAgd(ed=eddfdBdCZ\ejX[d=d>dDgid?dDdEiid@dFdEiidDgdDdEidGdDgdFdEidHdDgdDdEidFdEidIgd(ed=eddfdJdKZ]eeBe6jGe6j^e6j_e=e`gdLdMee ddfdNdOZaeeBe6jGe6j^e6j_e=e`gdLdMee ddfdPdQZbeeBe<ecgdRdSee ddfdTdUZddXdVdWZedS )Y    N)	overrides)givensettingsHealthCheck)DictSetcastUnionDefaultDictAnyList)	dataclass)ID
EmbeddingsIncludeIDsvalidate_embeddings)	ClientAPI)
Collection)
BundleRuleBasedStateMachineMultipleResultsrule
initializepreconditionconsumesrun_state_machine_as_testmultiple	invariant)defaultdict)resetNOT_CLUSTER_ONLY)wait_for_version_increaseget_collection_versionc                   C   s   dS )Nr    r$   r$   r$   e/home/air/biblejyuku/back/venv/lib/python3.10/site-packages/chromadb/test/property/test_embeddings.py<lambda>'   s    r&   traceskeyreturnc                 C   s   t |   d7  < d S )N   )r'   r(   r$   r$   r%   trace*      r,   c                  C   s(   t  D ]\} }t|  d|  qd S )Nz: )r'   itemsprint)r(   valuer$   r$   r%   print_traces/   s   r1   dtyper+   dtype_shared_st   i   )	min_value	max_value	dimensiondimension_shared_stc                   @   s    e Zd ZdZdZdZdZdZdS )EmbeddingStateMachineStatesr   add_embeddingsdelete_by_idsupdate_embeddingsupsert_embeddingsN)__name__
__module____qualname__r   r:   r;   r<   r=   r$   r$   r$   r%   r9   >   s    r9   T)with_hnsw_paramscollc                       s  e Zd ZU eed< edZee ed< def fddZ	e
eddejfddZ
ed1ddZeeeeddejd	ee fddZeejeedddded	d
fddZedd eejeedddddejd	d
fddZedd eejeeeejdddddejd	d
fdd Ze d1d!d"Ze d1d#d$Z e d1d%d&Z!e d1d'd(Z"dejd	d
fd)d*Z#d+e$e% d	d
fd,d-Z&d.e'd	d
fd/d0Z(  Z)S )2EmbeddingStateMachineBase
collectionembedding_idsclientc                    s"   t    || _tj| | _d S N)super__init__rF   
hypothesisstatefulRuleStrategy_rules_strategyselfrF   	__class__r$   r%   rI   N   s   
z"EmbeddingStateMachineBase.__init__rD   c                 C   sX   t | j | jj|j|j|jd| _|j| _td | t	j
 tjg g g g d| _d S )N)namemetadataembedding_functioninitids	metadatas	documents
embeddings)r    rF   create_collectionrS   rT   rU   rD   r,   on_state_changer9   r   
strategiesStateMachineRecordSetrecord_set_staterO   rD   r$   r$   r%   r   S   s   
z$EmbeddingStateMachineBase.initializer)   Nc                 C   s   | j | jj d S rG   )rF   delete_collectionrD   rS   rO   r$   r$   r%   teardownc   r-   z"EmbeddingStateMachineBase.teardowntarget
record_setrg   c                    sL  t d | tj t| t d dkrt d t d | j	d }t|dkrt
t d |} fdd|D } fdd|D  d rV fd	d|D nd  d
 rd fdd|D nd  d rr fdd|D nd d}| jjdi   | ttj| t|d  S | jjdi   | ttj  t d  S )Nr:   rX   r   add_more_embeddingsc                    s   g | ]	} d   |qS rX   )index.0idnormalized_record_setr$   r%   
<listcomp>|   s    z<EmbeddingStateMachineBase.add_embeddings.<locals>.<listcomp>c                       g | ]} d  | qS ri   r$   rl   irn   r$   r%   rp   ~       rY   c                    rq   )rY   r$   rr   rn   r$   r%   rp      rt   rZ   c                    rq   )rZ   r$   rr   rn   r$   r%   rp      rt   r[   c                    rq   )r[   r$   rr   rn   r$   r%   rp      rt   rW   r$   )r,   r]   r9   r:   
invariantswrap_alllensetintersectionr`   list
differencerD   add_upsert_embeddingsr   r^   	RecordSetr   )rO   rg   ry   new_idsindicesfiltered_record_setr$   rn   r%   r:   g   s<   z(EmbeddingStateMachineBase.add_embeddingsr*   min_sizeri   rX   c                    sF   t d  tj  fdd|D } jj|d  t| d S )Nzremove embeddingsc                    s   g | ]
} j d  |qS ri   )r`   rj   rk   rc   r$   r%   rp      s    z;EmbeddingStateMachineBase.delete_by_ids.<locals>.<listcomp>ri   )r,   r]   r9   r;   rD   delete_remove_embeddingsrx   )rO   rX   indices_to_remover$   rc   r%   r;      s
   z'EmbeddingStateMachineBase.delete_by_idsc                 C      t | jd dkS NrX      rw   r`   rc   r$   r$   r%   r&          z"EmbeddingStateMachineBase.<lambda>r   collection_strategyid_strategyr   max_sizerg   c                 C   4   t d | tj | jjdi | | | d S )Nzupdate embeddingsr$   )r,   r]   r9   r<   rD   updater}   rO   rg   r$   r$   r%   r<         
z+EmbeddingStateMachineBase.update_embeddingsc                 C   r   NrX      r   rc   r$   r$   r%   r&      r   c                 C   r   )Nzupsert embeddingsr$   )r,   r]   r9   r=   rD   upsertr}   r   r$   r$   r%   r=      r   z+EmbeddingStateMachineBase.upsert_embeddingsc                 C   s   t | jttj| j d S rG   )ru   countrD   r   r^   r~   r`   rc   r$   r$   r%   r      s   zEmbeddingStateMachineBase.countc                 C   s   t | j d S rG   )ru   no_duplicatesrD   rc   r$   r$   r%   r      s   z'EmbeddingStateMachineBase.no_duplicatesc                 C   s$   t j| jttj| jd| jd d S )Ngffffff?)rD   rg   
min_recallrU   )ru   ann_accuracyrD   r   r^   r~   r`   rU   rc   r$   r$   r%   r      s   
z&EmbeddingStateMachineBase.ann_accuracyc                 C   sD   t tj| j| _t| j| j t| j| j t| j| j d S rG   )	r   r^   r~   r`   ru   embeddings_matchrD   metadatas_matchdocuments_matchrc   r$   r$   r%   fields_match   s   z&EmbeddingStateMachineBase.fields_matchc                 C   s.  t |}t|d D ]\}}|| jd v r| jd |}|d d ur1|d | | jd |< n |d d us9J | jd us@J | |d | gd | jd |< |d d ur| jd | }|d urtttt	tt
tf f |}|d | d ur||d |  n	 n|d | | jd |< |d d ur|d | | jd |< q| jd | |d d ur| jd |d |  n!| jd usJ |d d usJ | jd | |d | gd  |d d ur| jd |d |  n| jd d  |d d ur| jd |d |  q| jd d  qd S )NrX   r[   rZ   r   rY   )ru   rv   	enumerater`   rj   rU   r   r   strr	   intfloatr   append)rO   rg   ro   idxrm   
target_idxr`   r$   r$   r%   r}      s   










z,EmbeddingStateMachineBase._upsert_embeddingsr   c                 C   sR   t |}|jdd |D ]}| jd |= | jd |= | jd |= | jd |= qd S )NT)reverserX   r[   rY   rZ   )rz   sortr`   )rO   r   indices_listrs   r$   r$   r%   r   !  s   z,EmbeddingStateMachineBase._remove_embeddings	new_statec                 C   s   d S rG   r$   )rO   r   r$   r$   r%   r]   +  s   z)EmbeddingStateMachineBase.on_state_changer)   N)*r>   r?   r@   r   __annotations__r   rE   r   r   rI   r   collection_str^   r   rd   r   
recordsetsr~   r   r:   stlistsr   r   r;   r   r<   one_of	safe_textr=   r   r   r   r   r   r}   r   r   r   r   r]   __classcell__r$   r$   rP   r%   rC   J   s\   
 '


I
rC   c                       sP  e Zd ZU edZee ed< def fddZe	e
ddejf fddZ	ed	d
 e d"ddZeeee
ddejdee f fddZeejeeddddeddf fddZedd
 eeje
edddddejddf fddZedd
 eeje
eeejdddddejddf fd d!Z  ZS )#EmbeddingStateMachinerE   rF   c                    s   t  | d S rG   )rH   rI   rN   rP   r$   r%   rI   2  s   zEmbeddingStateMachine.__init__rR   rD   c                    s@   t  | td| jjd  d| _t | _| j d | _	d S )Nz7[test_embeddings][initialize] Initialize collection id rm   r   version)
rH   r   r/   rD   _modellog_operation_countrx   unique_ids_in_log	get_modelcollection_versionra   rP   r$   r%   r   5  s   
z EmbeddingStateMachine.initializec                 C   s   t  o| jdkot| jdkS )N
   r   )r!   r   rw   r   rc   r$   r$   r%   r&   A  s    zEmbeddingStateMachine.<lambda>r)   Nc                 C   s   t | j| jj}|| jksJ || jkr$td || _d| _t | _d S td| t	| j| jj|dd}d| _t | _|| _d S )NzT[test_embeddings][wait_for_compaction] collection version has changed, so reset to 0r   zY[test_embeddings][wait_for_compaction] wait for version to increase from current version    )additional_time)
r#   rF   rD   rS   r   r/   r   rx   r   r"   )rO   current_versionnew_versionr$   r$   r%   wait_for_compaction@  s&   

z)EmbeddingStateMachine.wait_for_compactionre   rg   c                    sn   t  |}t|}td|d dt|d  |  jt|d 7  _|d D ]}|| jvr4| j| q'|S )Nz,[test_embeddings][add] Non Intersection ids rX    len )	rH   r:   ru   rv   r/   rw   r   r   r|   )rO   rg   resro   rm   rP   r$   r%   r:   `  s    

z$EmbeddingStateMachine.add_embeddingsr*   r   ri   rX   c                    sT   t  | td|dt| |  jt|7  _|D ]}|| jv r'| j| qd S )Nz[test_embeddings][delete] ids r   )rH   r;   r/   rw   r   r   remove)rO   rX   rm   rP   r$   r%   r;   u  s   
z#EmbeddingStateMachine.delete_by_idsc                 C   r   r   r   rc   r$   r$   r%   r&     r   r   r   r   c              	      sL   t  | td|d dtt|d  |  jtt|d 7  _d S )Nz[test_embeddings][update] ids rX   r   )rH   r<   r/   rw   ru   wrapr   r   rP   r$   r%   r<     s   
 z'EmbeddingStateMachine.update_embeddingsc                 C   r   r   r   rc   r$   r$   r%   r&     r   c              	      sv   t  | td|d dtt|d  |  jtt|d 7  _t|d D ]}|| jvr8| j| q+d S )Nz[test_embeddings][upsert] ids rX   r   )	rH   r=   r/   rw   ru   r   r   r   r|   )rO   rg   rm   rP   r$   r%   r=     s   

z'EmbeddingStateMachine.upsert_embeddingsr   )r>   r?   r@   r   rE   r   r   r   rI   r   r   r^   r   r   r   r   r   r~   r   r:   r   r   r   r   r;   r<   r   r   r=   r   r$   r$   rP   r%   r   /  sJ   
 



$r   caplogrF   c                    s6   |  tj t fddtdtjgdd t  d S )Nc                      s   t  S rG   )r   r$   rF   r$   r%   r&     s    z'test_embeddings_state.<locals>.<lambda>i_ )deadlinesuppress_health_check)r   )	set_levelloggingERRORr   r   r   filter_too_muchr1   )r   rF   r$   r   r%   test_embeddings_state  s   

r   c                 C   s  t | }|jtjdddddd t dtji g ddd
d |  |	  |
  |  |jg d	d
dgddgddgddgddgddggg dd dd\}}}}}}|  |	  |
  |  |j|||||gd tsv|  |  |	  |
  |  |  d S )NA00   hnsw:construction_efhnsw:search_efhnsw:Mr4   FT
rS   rT   rU   rm   r7   r2   known_metadata_keysknown_document_keywordshas_documentshas_embeddingsrR   )012345      ?     ?     P?     ?     ÿ     ?          ?     ?     Ϳ)NNNNNNrX   r[   rY   rZ   r   ri   )r   r   r^   r   uuiduuid4npfloat16r   r   r   r   r:   r;   r!   r   rd   )rF   statev1v2v3v4v5v6r$   r$   r%   test_add_then_delete_n_minus_1  s^   r   c                 C   s   t |}|jtjdddddd tddtji g ddd
d	 |  |	  |
  |  |jg d
ddgddgddgddgddggg dd dd\}}}}}|  |	  |
  |  |j|gddgg||igd dd |  |  d S )Nr   r   r   z$2fb0c945-b877-42ab-9417-bfe0f6b172afr4   FTr   rR   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   )NNNNNr   r   g     ?g     ?)r   r   r^   r   r   UUIDr   r   r   r   r   r   r:   r<   rd   )r   rF   r   r   r   r   r   r   r$   r$   r%   test_update_none  s^   r   c                 C   s  t | }|jtjdi dddddddd	d
dddddddddddddddddddddddd tddtji g ddd
d  |  |	  |
  |  |jg d!g d"g d#g d$g d%g d&g d'gd d d(d)}d*}i }|D ]
}|||< |d+7 }q|  |	  |
  |  |j|d* |d, d-|d. d/gg d0g d1g d2g d3g d4gd d d5d) |  |	  |
  |  ts|  |  |	  |
  |  |jd6|d* |d, gg d7g d8g d9gd d d(d) |  |	  |
  |  |j|d: |d+ |d. |d, |d gd; |  |	  |
  |  |jd<d=d>d?|d+ gg d@g dAg dBg dCg dDgd d d5d)}dE}|D ]}|||< |d+7 }q[|  |	  |
  ts||  d S d S )FNKR3cfUfmxsi3g    ~.AbMMvvrqM4MKmp5CJB8Ai  -T37PNiVkn5KZfkpod3ND5soL_KA4zcZL9lRN9   Oc8G7ysXmE8lp4Hos_POQe8Unz1uJBI930U   teFtyMg      R0ZiZmIOwir   r   r   r   z$284b6e99-b19e-49b2-96a4-a2a93a95447dr   r   rR   )255lz3-rs   Nk9yPvT)g333333?gffffff@g      ?)      @      @r4   )r*   r4   r   )   r      )g!@g      "@   )r  r  ffffff@r   r   r   r*   r  KWcDaHUVD6MxEiJr   PdlP1d6w)r*      r  )r   r   	   )r  r   r   )r   r     )r*   r   r  )rX   r[   rZ   rY   TpjiboLSuYWBJDbRW1zeNmC)r  r  r  )r  r  r   )r*   r   r  r4   ri   oD3V84RtTDwlc9C8_evn)r  g@g(\	@)gHzG?gQ@gp=
ף@)g#@g#@g?)gffffff#@r  r  )g333333@r  gL#@r  )r   r   r^   r   r   r   r   float32r   r   r   r   r:   r=   r!   r   r;   )rF   r   r[   rs   emb_list	embeddingr$   r$   r%   test_add_delete_add!  s  	

  r!  c                 C   s   t |  | jdd}|jdgdggd | dksJ |jdgdggd | dks.J | }|d dgks;J |jdgd | d	ksJJ d S )
NfoorS   a        rX   r[   r*   rX   ri   r   )r    r\   r|   r   getr   )rF   rB   resultsr$   r$   r%   test_multi_add  s   r)  c                 C   s   t |  | jdd}ttj |jddgdgdggd W d    n1 s(w   Y  ttj |jddgdgdggd W d    d S 1 sLw   Y  d S )Nr"  r#  r$  r%  g?r&  )r    r\   pytestraiseserrorsDuplicateIDErrorr|   r   )rF   rB   r$   r$   r%   test_dup_add  s   "r.  c                 C   s   t |  | jdd}g d}tjdd}tjdd}|jtj||f |d}|D ]}|| }|d us9J tdd	 |D sDJ q-d S )
Nr"  r#  )rZ   rY   r[   	distancesr*   i  d   )query_embeddingsincludec                 S   s   g | ]}t |d kqS )r   )rw   )rl   resultr$   r$   r%   rp     rt   z*test_query_without_add.<locals>.<listcomp>)r    r\   r   randomrandintquerytolistall)rF   rB   fieldsNKr(  fieldfield_resultsr$   r$   r%   test_query_without_add  s   r>  c                 C   s|   t |  | jdd}|jdgg dd}t|d dksJ t|d dks(J t|d	 dks2J t|d
 dks<J d S )Nr"  r#  r$  )rZ   rY   r[   )rX   r2  rX   r   rY   rZ   r[   )r    r\   r'  rw   )rF   rB   r3  r$   r$   r%   test_get_non_existent  s   r?  z9We don't properly escape SQL internally, causing problems)reasonc                 C   s^   t |  d}| jdd}|j|gdggd | dksJ |j|gd | dks-J d S )	Nr"  r#  r%  r&  r*   ri   r   )r    r\   r|   r   r   )rF   rm   rB   r$   r$   r%   test_escape_chars_in_ids  s   rB  kwargsrX   wherewhere_document)rE  rD  c                 C   sb   t |  | jdd}tt}|jdi | W d    n1 s"w   Y  dt|v s/J d S )Nr"  r#  z@You must provide either ids, where, or where_document to delete.r$   )r    r\   r*  r+  	Exceptionr   r   )rF   rC  rB   er$   r$   r%   test_delete_empty_fails  s   rH  r"  barz	$contains)rX   rD  )rX   rE  )rX   rD  rE  c                 C   s(   t |  | jdd}|jdi | d S )Nr"  r#  r$   )r    r\   r   )rF   rC  rB   r$   r$   r%   test_delete_success  s   rJ  )supported_typesrK  c                 C   6   t dd| }tt|}tdd |D sJ d S )Nr   c                 S   (   g | ]}t |totd d |D qS )c                 S   &   g | ]}t |ttfot |t qS r$   
isinstancer   r   boolrl   vecr$   r$   r%   rp         zXtest_autocasting_validate_embeddings_for_compatible_types.<locals>.<listcomp>.<listcomp>rP  rz   r8  rl   r0   r$   r$   r%   rp         
zMtest_autocasting_validate_embeddings_for_compatible_types.<locals>.<listcomp>)r^   create_embeddingsr   r   _normalize_embeddingsr8  rK  embdsvalidated_embeddingsr$   r$   r%   9test_autocasting_validate_embeddings_for_compatible_types     r]  c                 C   rL  )Nr   c                 S   rM  )c                 S   rN  r$   rO  rR  r$   r$   r%   rp   )  rT  zPtest_autocasting_validate_embeddings_with_ndarray.<locals>.<listcomp>.<listcomp>rU  rV  r$   r$   r%   rp   &  rW  zEtest_autocasting_validate_embeddings_with_ndarray.<locals>.<listcomp>)r^   create_embeddings_ndarrayr   r   rY  r8  rZ  r$   r$   r%   1test_autocasting_validate_embeddings_with_ndarray  r^  r`  )unsupported_typesra  c                 C   sZ   t dd| }tt}tt| W d    n1 sw   Y  dt|v s+J d S )Nr   z9Expected each value in the embedding to be a int or float)	r^   rX  r*  r+  
ValueErrorr   r   rY  r   )ra  r[  rG  r$   r$   r%   7test_autocasting_validate_embeddings_incompatible_types3  s
   rc  c                  C   sL   g g} t t}t|  W d    n1 sw   Y  dt|v s$J d S )Nz@Expected each embedding in the embeddings to be a non-empty list)r*  r+  rb  r   r   )r[  rG  r$   r$   r%   test_0dim_embedding_validation>  s
   
rd  r   )fhypothesis.statefulrJ   hypothesis.strategiesr   r*  r   r^   r   r   r   r   typingr   r   r   r	   r
   r   r   dataclassesr   chromadb.api.typesr   r   r   r   r   chromadb.errorsr,  chromadb.apir   chromadb.api.models.Collectionr   !chromadb.test.property.strategiestestpropertyr   r   r   r   r   r   r   r   r   r   collectionsr   !chromadb.test.property.invariantsru   chromadb.test.conftestr    r!   numpyr   r   -chromadb.test.utils.wait_for_version_increaser"   r#   r'   r   r   r   r,   r1   sharedsampled_fromfloat_typesr3   SearchStrategyr   r  float64integersr8   r9   r   rC   r   LogCaptureFixturer   r   r   r!  r)  r.  r>  r?  markxfailrB  parametrizerH  rJ  int32int64r   r]  r`  rQ  rc  rd  r$   r$   r$   r%   <module>   s   
 $0
 f|56 	




