o
    3IhP                     @   sd  d dl mZ d dlmZmZmZmZ d dlmZ d dl	m	Z	 d dl
Z
d dlmZmZmZmZ d dlmZmZmZmZmZmZ d dlm  mZ d dlmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9 d d	l:m;Z;m<Z< d dl=Z=d d
l>m?Z? d dl@mAZAmBZBmCZCmDZD G dd de;eZEdS )    )futures)AnyDictListcast)UUID)	overridesN)DEFAULT_DATABASEDEFAULT_TENANT	ComponentSystem)from_proto_metadatafrom_proto_update_metadatafrom_proto_segmentfrom_proto_segment_scopeto_proto_collectionto_proto_segment)CreateCollectionRequestCreateCollectionResponseCreateDatabaseRequestCreateDatabaseResponseCreateSegmentRequestCreateSegmentResponseCreateTenantRequestCreateTenantResponseCountCollectionsRequestCountCollectionsResponseDeleteCollectionRequestDeleteCollectionResponseDeleteSegmentRequestDeleteSegmentResponseGetCollectionsRequestGetCollectionsResponseGetCollectionSizeRequestGetCollectionSizeResponse GetCollectionWithSegmentsRequest!GetCollectionWithSegmentsResponseGetDatabaseRequestGetDatabaseResponseGetSegmentsRequestGetSegmentsResponseGetTenantRequestGetTenantResponseResetStateResponseUpdateCollectionRequestUpdateCollectionResponseUpdateSegmentRequestUpdateSegmentResponse)SysDBServiceradd_SysDBServicer_to_server)Empty)
CollectionMetadataSegmentSegmentScopec                       s,  e Zd ZU dZejed< eed< i Ze	e
ef ed< i Ze	e
ee
 f ed< i Ze	e
e	e
e	e
ef f f ed< i Ze	e
e	e
ef f ed< def fd	d
Zed> fddZed> fddZed> fddZedddedejdefddZedddedejdefddZedddedejde fddZ!eddde"dejde#fddZ$eddde%dejde&fdd Z'd!edejde&fd"d#Z(eddde)dejde*fd$d%Z+eddde,dejde-fd&d'Z.eddde/dejde0fd(d)Z1eddde2dejde3fd*d+Z4eddde5dejde6fd,d-Z7eddde8dejde9fd.d/Z:eddde;dejde<fd0d1Z=eddde>dejde?fd2d3Z@edddeAdejdeBfd4d5ZCedddeDdejdeEfd6d7ZFedddeGdejdeHfd8d9ZId:eJd;eKjLddfd<d=ZM  ZNS )?GrpcMockSysDBzA mock sysdb implementation that can be used for testing the grpc client. It stores
    state in simple python data structures instead of a database._server_server_port	_segments_collection_to_segments$_tenants_to_databases_to_collections_tenants_to_database_to_idsystemc                    s   |j d| _t |S )Nchroma_server_grpc_port)settingsrequirer;   super__init__)selfr@   	__class__ \/home/air/sanwanet/gpt-api/venv/lib/python3.10/site-packages/chromadb/db/impl/grpc/server.pyrE   H   s   zGrpcMockSysDB.__init__returnNc                    sH   t tjdd| _t| | j | jd| j  | j  t	  S )N
   )max_workersz[::]:)
grpcserverr   ThreadPoolExecutorr:   r3   add_insecure_portr;   startrD   rF   rG   rI   rJ   rR   L   s
   

zGrpcMockSysDB.startc                    s   | j d  t  S N)r:   stoprD   rS   rG   rI   rJ   rU   T   s   
zGrpcMockSysDB.stopc                    sL   i | _ i | _i | jt< i | jt t< i | jt< tdd| jt t< t  S )Nr   )int)r<   r>   r
   r	   r?   r   rD   reset_staterS   rG   rI   rJ   rW   Y   s   


zGrpcMockSysDB.reset_stateF)check_signaturerequestcontextc                 C   s~   |j }|j}|| jvr|tjjd| d || j| v r*|tjjd| d i | j| |< t|j	d| j
| |< t S )NTenant 
 not found	Database  already existshex)tenantnamer>   abortrN   
StatusCode	NOT_FOUNDALREADY_EXISTSr   idr?   r   )rF   rY   rZ   ra   databaserI   rI   rJ   CreateDatabased   s   
zGrpcMockSysDB.CreateDatabasec                 C   sz   |j }|j}|| jvr|tjjd| d || j| vr*|tjjd| d | j| | }tt	j
|j||ddS )Nr[   r\   r]   )rg   rb   ra   )rh   )ra   rb   r>   rc   rN   rd   re   r?   r(   protoDatabaser`   )rF   rY   rZ   ra   rh   rg   rI   rI   rJ   GetDatabaset   s   
zGrpcMockSysDB.GetDatabasec                 C   sB   |j }|| jv r|tjjd| d i | j|< i | j|< t S )Nr[   r^   )rb   r>   rc   rN   rd   rf   r?   r   rF   rY   rZ   ra   rI   rI   rJ   CreateTenant   s   


zGrpcMockSysDB.CreateTenantc                 C   s:   |j }|| jvr|tjjd| d ttj|ddS )Nr[   r\   rb   )ra   )	rb   r>   rc   rN   rd   re   r,   rj   Tenantrm   rI   rI   rJ   	GetTenant   s   

zGrpcMockSysDB.GetTenantc                 C   s   t |j}| ||S rT   )r   segmentCreateSegmentHelper)rF   rY   rZ   rr   rI   rI   rJ   CreateSegment   s   
zGrpcMockSysDB.CreateSegmentrr   c                 C   sB   |d j | jv r|tjjd|d  d || j|d j < t S )Nrg   Segment r^   )r`   r<   rc   rN   rd   rf   r   )rF   rr   rZ   rI   rI   rJ   rs      s   z!GrpcMockSysDB.CreateSegmentHelperc                 C   s:   |j }|| jv r| j|= t S |tjjd| d d S )Nru   r\   )rg   r<   r    rc   rN   rd   re   )rF   rY   rZ   id_to_deleterI   rI   rJ   DeleteSegment   s   
zGrpcMockSysDB.DeleteSegmentc           	      C   s   | drt|jdnd }| dr|jnd }| dr!t|jnd }t|jd}g }| j D ]+}|r;|d |kr;q0|rD|d |krDq0|rM|d |krMq0|rV|d |krVq0|	| q0t
dd |D dS )	Nrg   r_   typescope
collectionc                 S      g | ]}t |qS rI   r   .0rr   rI   rI   rJ   
<listcomp>       z-GrpcMockSysDB.GetSegments.<locals>.<listcomp>)segments)HasFieldr   rg   rx   r   ry   rz   r<   valuesappendr*   )	rF   rY   rZ   	target_idtarget_typetarget_scopetarget_collectionfound_segmentsrr   rI   rI   rJ   GetSegments   s*   zGrpcMockSysDB.GetSegmentsc                 C   s   t |j}|j| jvr|tjjd| d d S | j|j }|dr@t	t
ttf |d }|d d u r9i |d< | ||j |drL|jrLi |d< t S )Nru   r\   metadatareset_metadata)r   rg   r`   r<   rc   rN   rd   re   r   r   r   strr   _merge_metadatar   r   r1   )rF   rY   rZ   id_to_updaterr   targetrI   rI   rJ   UpdateSegment   s   

zGrpcMockSysDB.UpdateSegmentc                    sT  |j  |j}|j}|| jvr|tjjd| d || j| vr-|tjjd| d | j D ]D\}}| D ];\}}|j	|v ru||jksM||jkr`|tjj
d|j	 d| d|  q:|jsu|tjj
d|j	 d| d|  q:q2| j| | }	 fdd|	 D }
t|
d	ksJ t|
d
kr|jr|
d
 }tt|ddS |tjj
d  d t|j}t|j	d}t||j |t|j|j||d
d}g }|jD ]8}t|}|d j| jv r|D ]}| t|d| q|tjj
d|d  d | || ||d j q||	|j	< | d| d|j	 }|| j |< tt|ddS )Nr[   r\   r]   Collection z already exists in tenant z
 database c                    s   g | ]
}|d   kr|qS ro   rI   )r~   ccollection_namerI   rJ   r     s    z2GrpcMockSysDB.CreateCollection.<locals>.<listcomp>   r   F)rz   createdr^   r_   )rg   rb   configuration_jsonr   	dimensionrh   ra   versionrg   )rg   ru   :T)!rb   ra   rh   r>   rc   rN   rd   re   itemsrg   rf   get_or_creater   lenr   r   jsonloadsconfiguration_json_strr   r5   r   r   r   r   r   r`   r<   rw   r   rs   r   r=   )rF   rY   rZ   ra   rh   search_tenant	databasessearch_databasesearch_collectionscollectionsmatchesexisting_collectionr   rg   new_collectionsegments_addedsegment_protorr   scollection_unique_keyrI   r   rJ   CreateCollection   s   







zGrpcMockSysDB.CreateCollectionc           
      C   s   |j }|j}|j}|| jvr|tjjd| d || j| vr-|tjjd| d | j| | }||v rX||= | d| d| }| j| }|rU|D ]}	| j	|	= qNt
 S |tjjd| d d S )Nr[   r\   r]   r   r   )rg   ra   rh   r>   rc   rN   rd   re   r=   r<   r   )
rF   rY   rZ   collection_idra   rh   r   r   segment_ids
segment_idrI   rI   rJ   DeleteCollectionG  s&   


zGrpcMockSysDB.DeleteCollectionc              	   C   s   | drt|jdnd }| dr|jnd }i }| j D ]5\}}| D ],\}}	|jdkr5||jkr5q&|jdkr@||jkr@q&||	 t	d| d| d|	  q&qg }
|
 D ]}|re|d |kreqZ|rn|d |krnqZ|
| qZtdd	 |
D d
S )Nrg   r_   rb    Tenant: , Database: , Collections: c                 S   r{   rI   )r   )r~   rz   rI   rI   rJ   r   z  s    z0GrpcMockSysDB.GetCollections.<locals>.<listcomp>)r   )r   r   rg   rb   r>   r   ra   rh   updateprintr   r   r"   )rF   rY   rZ   r   target_nameallCollectionsra   r   rh   r   found_collectionsrz   rI   rI   rJ   GetCollections`  s4   
	zGrpcMockSysDB.GetCollectionsc                 C   s,   t |j|jd}| ||}tt|jdS )N)ra   rh   )count)r!   ra   rh   r   r   r   r   )rF   rY   rZ   r   rI   rI   rJ   CountCollections  s   zGrpcMockSysDB.CountCollectionsc                 C   s
   t ddS )Nr   )total_records_post_compaction)r$   rF   rY   rZ   rI   rI   rJ   GetCollectionSize  s   zGrpcMockSysDB.GetCollectionSizec              	      s  i } j  D ]\}}| D ]\}}|| td| d| d|  qq||jd }|d u r?|tjj	d|j d |j
 d|j d|j }	 fdd j|	 D }
d	d
 |
D tjtjtjhkru|tjjd| d|
  tt|dd |
D dS )Nr   r   r   zCollection with id r\   r   c                    s   g | ]} j | qS rI   )r<   )r~   rg   rS   rI   rJ   r     s    z;GrpcMockSysDB.GetCollectionWithSegments.<locals>.<listcomp>c                 S   s   h | ]}|d  qS )ry   rI   r}   rI   rI   rJ   	<setcomp>  r   z:GrpcMockSysDB.GetCollectionWithSegments.<locals>.<setcomp>z#Incomplete segments for collection z: c                 S   r{   rI   r|   r}   rI   rI   rJ   r     r   )rz   r   )r>   r   r   r   getrg   rc   rN   rd   re   ra   rh   r=   r8   METADATARECORDVECTORINTERNALr&   r   )rF   rY   rZ   r   ra   r   rh   r   rz   r   r   rI   rS   rJ   GetCollectionWithSegments  s>   

z'GrpcMockSysDB.GetCollectionWithSegmentsc                 C   s  t |j}i }| j D ]\}}| D ]\}}|j|v r|}qq|j|vr4|tjjd| d d S ||j }	|	drC|j
|	d< |	drM|j|	d< |	drwt|j}
d }|
d urpi }|
 D ]\}}|d uro|||< qc||	d< t S |	dr|jri |	d< t S )Nr   r\   rb   r   r   r   )r   rg   r>   r   r`   rc   rN   rd   re   r   rb   r   r   r   r   r/   )rF   rY   rZ   r   r   ra   r   rh   maybe_collectionsrz   update_metadatacleaned_metadatakeyvaluerI   rI   rJ   UpdateCollection  s@   










zGrpcMockSysDB.UpdateCollectionc                 C   s   |    t S rT   )rW   r-   r   rI   rI   rJ   
ResetState  s   zGrpcMockSysDB.ResetStater   sourcec                 C   s^   t tttf |}t tttf t|}|| | D ]\}}|d u r,||v r,||= qd S rT   )r   r   r   r   r   r   r   )rF   r   r   target_metadatasource_metadatar   r   rI   rI   rJ   r     s   
zGrpcMockSysDB._merge_metadata)rK   N)O__name__
__module____qualname____doc__rN   Server__annotations__rV   r<   r   r   r7   r=   r   r>   r5   r?   r   r   rE   r   rR   rU   rW   r   ServicerContextr   ri   r'   r(   rl   r   r   rn   r+   r,   rq   r   r   rt   rs   r   r    rw   r)   r*   r   r0   r1   r   r   r   r   r   r   r   r!   r"   r   r   r   r   r#   r$   r   r%   r&   r   r.   r/   r   r4   r-   r   r6   rj   UpdateMetadatar   __classcell__rI   rI   rG   rJ   r9   ;   s4  
 


Y
%) r9   )F
concurrentr   typingr   r   r   r   uuidr   r   r   chromadb.configr	   r
   r   r   chromadb.proto.convertr   r   r   r   r   r   chromadb.proto.chroma_pb2rj   
chroma_pb2chromadb.proto.coordinator_pb2r   r   r   r   r   r   r   r   r   r   r   r   r   r    r!   r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r0   r1   #chromadb.proto.coordinator_pb2_grpcr2   r3   rN   google.protobuf.empty_pb2r4   chromadb.typesr5   r6   r7   r8   r9   rI   rI   rI   rJ   <module>   s     !