o
    1Xxi}                     @   s"  d dl Z d dlmZmZmZmZmZmZmZm	Z	 d dl
mZ d dlmZ d dlmZmZ d dlmZ d dlmZmZmZmZ d dlmZmZmZ d d	lmZmZmZm Z m!Z!m"Z" d d
l#m$Z$ d dl%m&Z&m'Z'm(Z(m)Z) d dl*m+Z+ d dl,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5 G dd dee$Z6dS )    N)OptionalSequenceAnyTuplecastDictUnionSet)UUID)override)TableColumn)groupby)CollectionConfigurationInternalConfigurationParameterHNSWConfigurationInternalInvalidConfigurationError)DEFAULT_DATABASEDEFAULT_TENANTSystem)CursorSqlDBParameterValueget_sqlNotFoundErrorUniqueConstraintError)SysDB)add_attributes_to_current_spanOpenTelemetryClientOpenTelemetryGranularitytrace_method)Producer)	DatabaseOptionalArgumentSegmentMetadata
CollectionSegmentScopeTenantUnspecifiedUpdateMetadatac                       sL  e Zd ZU eed< def fddZedej	e
d@ fdd	Ze
efd
edededdfddZe
efdededefddZe
deddfddZe
dedefddZe
deddfddZedej	e
dddeefd
edededee dee dedededeeef fddZedej	e
				dAd
ee d ee d!ee  d"ee de!e f
d#dZ"ed$ej	e
ddeeddfd
ee dee deded%ee d&ee de!e fd'd$Z#ed(ej	e
d
eddfd)d(Z$ed*ej	e
eefd
edededdfd+d*Z%ed,ej	e
e& e& fd
ed"e'ee  de'ee(  ddfd-d,Z)ed.ej	e
e& e& e& fd
ede'e de'ee  de'ee(  ddf
d/d.Z*ed0ej	d1e!ee+d2f  dee fd3d0Z,ed4ej		dBd5e-d6e.d7e/d
ede(d8ee0e  ddfd9d4Z1d:ed;edefd<d=Z2d:e+dee defd>d?Z3  Z4S )CSqlSysDB	_producersystemc                    s   t  | |t| _d S N)super__init__requirer   _opentelemetry_client)selfr-   	__class__ W/home/air/biblejyuku/back/venv/lib/python3.10/site-packages/chromadb/db/mixins/sysdb.pyr0   1   s   zSqlSysDB.__init__zSqlSysDB.create_segmentreturnNc                    s   t    | jt| _d S r.   )r/   start_systeminstancer!   r,   )r3   r4   r6   r7   r9   5   s   
zSqlSysDB.startidnametenantc                 C   s   |   h}td}td}|  ||j|j|jt	| 
|t	||  |j||jt	|k}t||  \}}	z|||	 W n |  yc }
 ztd| d| |
d }
~
ww W d    d S 1 sow   Y  d S )N	databasestenants	Database z already exists for tenant )txr   querybuilderintocolumnsr<   r=   	tenant_idinsertr   
uuid_to_dbselectfrom_wherer   parameter_formatexecuteunique_constraint_errorr   )r3   r<   r=   r>   curr?   r@   insert_databasesqlparamser6   r6   r7   create_database;   s8   
"zSqlSysDB.create_databasec           
      C   s   |   i}td}|  ||j|j|jt|k|j	t|k}t
||  \}}||| }|sCtd| d| |d d u rStd| d| tt| |d }	t|	|d |dW  d    S 1 spw   Y  d S )Nr?   rA   z not found for tenant r      )r<   r=   r>   )rB   r   rC   rJ   rI   r<   r=   rK   r   rF   r   rL   rM   fetchoner   r   r
   uuid_from_dbr"   )
r3   r=   r>   rO   r?   qrQ   rR   rowr<   r6   r6   r7   get_databaseX   s*   
$zSqlSysDB.get_databasec                 C   s   |   F}td}|  ||jt|}t|| 	 \}}z|
|| W n |  yA } z	td| d|d }~ww W d    d S 1 sMw   Y  d S )Nr@   Tenant  already exists)rB   r   rC   rD   rE   r<   rG   r   r   rL   rM   rN   r   )r3   r=   rO   r@   insert_tenantrQ   rR   rS   r6   r6   r7   create_tenantp   s    

"zSqlSysDB.create_tenantc                 C   s   |   ?}td}|  ||j|jt|k}t|| 	 \}}|
|| }|s7td| dt|dW  d    S 1 sFw   Y  d S )Nr@   r[   
 not found)r=   )rB   r   rC   rJ   rI   r<   rK   r   r   rL   rM   rV   r   r(   )r3   r=   rO   r@   rX   rQ   rR   rY   r6   r6   r7   
get_tenant   s   
$zSqlSysDB.get_tenantsegmentc           	      C   sP  t t|d |d |d jt|d d |  }td}|  ||j|j	|j
|jt| |d t|d t|d jt| |d }t||  \}}z||| W n |  yw } ztd|d  d|d }~ww td	}|d
 r| |||j|d |d
  W d    d S W d    d S 1 sw   Y  d S )Nr<   typescope
collection
segment_idsegment_typesegment_scoperd   segmentsSegment r\   segment_metadatametadata)r   strvaluerB   r   rC   rD   rE   r<   rb   rc   rd   rG   r   rH   r   rL   rM   rN   r   _insert_metadatarf   )	r3   ra   rO   ri   insert_segmentrQ   rR   rS   
metadata_tr6   r6   r7   create_segment   s^   



"zSqlSysDB.create_collectionFconfigurationrl   	dimensionget_or_createdatabasec	                 C   s  |d u r
|s
t dtt||d | j|||d}	|	rK|rC|	d }
|d ur5|
d |kr5| j|
j|d | j|
d ||dd d	fS td
| dt|||||||dd}
|  }t	d}t	d}| 
 ||j|j|j|j|jt| |
d t|
d t| t|
d | 
 |j||jt|k|jt|k}t||  \}}z||| W n |  y } ztd
|
d  d|d }~ww t	d}|
d r| |||j|
j|
d  W d    |
dfS W d    |
dfS 1 sw   Y  |
dfS )Nz.id must be specified if get_or_create is Falsecollection_idcollection_name)r=   r>   rv   r   rl   )rl   r<   )r<   r>   rv   FCollection r\   r<   r=   rs   rl   rt   r>   rv   versioncollectionsr?   r=   rt   collection_metadataT)
ValueErrorr   rm   get_collectionsupdate_collectionr<   r   r&   rB   r   rC   rD   rE   r=   config_json_strrt   database_idrG   r   rH   to_json_strrI   rJ   rK   rF   r   rL   rM   rN   ro   rx   )r3   r<   r=   rs   rl   rt   ru   r>   rv   existingrd   rO   r}   r?   insert_collectionrQ   rR   rS   rq   r6   r6   r7   create_collection   s   





+
++zSqlSysDB.get_segmentsrb   rc   rd   c                 C   s  t t||r|nd|r|jndt|d td}td}|  ||j|j|j	|j
|j|j|j|j|j	||j|jk|j}|rV||jt| |k}|rb||jt|k}|ro||j	t|jk}|r~||j
t| |k}|  m}t||  \}	}
||	|
 }t|dd }g }|D ]F\}}| t|}t|}t|d d }tt|d d	 }|d d
 r| |d d
 nd }|  |}|!t"t#t$|||||d q|W  d    S 1 sw   Y  d S )N re   ri   rk   c                 S      t t| d S Nr   r   objectrr6   r6   r7   <lambda>R      z'SqlSysDB.get_segments.<locals>.<lambda>r   rU         )r<   rb   rc   rd   rl   )%r   rm   rn   r   rC   rJ   rI   r<   rb   rc   rd   key	str_value	int_valuefloat_value
bool_value	left_joinonrf   orderbyrK   r   rH   rB   r   rL   rM   fetchallr   rW   listr'   _metadata_from_rowsappendr$   r   r
   )r3   r<   rb   rc   rd   
segments_trq   rX   rO   rQ   rR   rows
by_segmentri   rf   segment_rowsrl   r6   r6   r7   get_segments  sv   	

"

$zSqlSysDB.get_collectionslimitoffsetc                 C   s  |dur|du s|du rt dtt||r|ndd td}td}td}	|  ||j|j|j	|j
|	j|	j|j|j|j|j|j||j|jk|	|j|	jk|j}
|ro|
|jt| |k}
|r{|
|jt|k}
|du r|r|rtd}	|
|j|  |	j|	|	jt|k|	jt|kk}
|  }t|
|  \}}||| }t|dd	 }g }|D ]f\}}| t|}t |}t|d
 d }| !|}|d
 d rt"|d
 d nd}|d
 d dur| #t||d
 d }n| $||}|%t&t't(|||||t|d
 d t|d
 d d
d q|durE|du r<d
}||||  }n||d }|W  d   S 1 sXw   Y  dS )z;Get collections by name, embedding function and/or metadataNzmIf name is specified, tenant and database must also be specified in order to uniquely identify the collectionr   rw   r}   r~   r?   c                 S   r   r   r   r   r6   r6   r7   r     r   z*SqlSysDB.get_collections.<locals>.<lambda>r   rU   r   r         r{   ))r   r   rm   r   rC   rJ   rI   r<   r=   r   rt   rF   r   r   r   r   r   r   r   rx   r   r   rK   r   rH   rB   r   rL   rM   r   r   rW   r   r   int&_load_config_from_json_str_and_migrate!_insert_config_from_legacy_paramsr   r&   r   r
   )r3   r<   r=   r>   rv   r   r   collections_trq   databases_trX   rO   rQ   rR   r   by_collectionr}   rx   collection_rowsrl   rt   rs   r6   r6   r7   r   g  s   



 

&zSqlSysDB.delete_segmentc                 C   s   t dt|i td}|  ||jt| |k	 }| 
 (}t||  \}}|d }||| }|sDtd| dW d   dS 1 sOw   Y  dS )zDelete a segment from the SysDBrf   ri    RETURNING idrj   r_   N)r   rm   r   rC   rJ   rK   r<   r   rH   deleterB   r   rL   rM   rV   r   )r3   r<   trX   rO   rQ   rR   resultr6   r6   r7   delete_segment  s$   
"zSqlSysDB.delete_collectionc                 C   s   t dt|i td}td}|  ||jt| |k|j	|  
|j||jt|k|jt|kk }|  '}t||  \}}	|d }|||	 }
|
sgtd| dW d   n1 sqw   Y  | j|
d  dS )	z{Delete a collection and all associated segments from the SysDB. Deletes
        the log stream for this collection as well.rx   r}   r?   r   rz   r_   Nr   )r   rm   r   rC   rJ   rK   r<   r   rH   r   rI   r=   rF   r   rB   r   rL   rM   rV   r   r,   
delete_log)r3   r<   r>   rv   r   r   rX   rO   rQ   rR   r   r6   r6   r7   delete_collection  s:   

zSqlSysDB.update_segmentc           
      C   sv  t t|t|d td}td}|  ||jt| |k}|t	 ks=t
tt |}||jt| |}|  p}t||  \}}	|rS|||	 |d u r{|  ||jt| |k }t||  \}}	|||	 n.|t	 krt
t|}t
t|}| |||j||t|  W d    d S W d    d S W d    d S 1 sw   Y  d S )N)rf   rd   ri   rk   )r   rm   r   rC   updaterK   r<   r   rH   r)   r   r   r
   setrd   rB   r   rL   rM   rJ   rf   r   r*   ro   keys)
r3   r<   rd   rl   r   rq   rX   rO   rQ   rR   r6   r6   r7   update_segment  sX   





"zSqlSysDB.update_collectionc                 C   s  t dt|i td}td}|  ||jt| |k}|t	 ks0|
|jt|}|t	 ks>|
|jt|}|  z}t||  \}	}
|	rd|	d }	||	|
}| sdtd| d|t	 kr|  ||jt| |k }t||  \}	}
||	|
 |d urtt|}| |||j||t
|  W d    d S W d    d S W d    d S 1 sw   Y  d S )Nrx   r}   r~   r   rz   r_   )r   rm   r   rC   r   rK   r<   r   rH   r)   r   r=   rt   rB   r   rL   rM   rV   r   rJ   rx   r   r   r*   ro   r   )r3   r<   r=   rt   rl   r   rq   rX   rO   rQ   rR   r   r6   r6   r7   r   R  s\   	





"zSqlSysDB._metadata_from_rowsr   .c                 C   s   t dt|i i }|D ]C}t|d }|d dur#t|d ||< q|d dur2t|d ||< q|d durAt|d ||< q|d durOt|d ||< q|pSdS )zGiven SQL rows, return a metadata map (assuming that the last four columns
        are the key, str_value, int_value & float_value)num_rowsN)r   lenrm   r   floatbool)r3   r   rl   rY   r   r6   r6   r7   r     s"   zSqlSysDB._insert_metadatarO   tableid_col
clear_keysc                 C   s  t dt|i |r8|  ||t| |k|jdd |D 	 }t
||  \}}	|||	 |  |||j|j|j|j|j}| |}
| D ]c\}}t|trq|t|
t|d d d tt|}qUt|tr|t|
t|t|d d d }qUt|tr|t|
t|d t|d d }qUt|tr|t|
t|d d t|d }qU|d u rqUqUt
||  \}}	|r|||	 d S d S )Nnum_keysc                 S   s   g | ]}t |qS r6   )r   ).0kr6   r6   r7   
<listcomp>  s    z-SqlSysDB._insert_metadata.<locals>.<listcomp>)r   r   rC   rJ   rK   r   rH   r   isinr   r   rL   rM   rD   rE   r   r   r   r   items
isinstancer   rG   r   rm   r   )r3   rO   r   r   r<   rl   r   rX   rQ   rR   sql_idr   vr6   r6   r7   ro     s   





rx   json_strc                 C   sB  zt |}W n t jy   td| w zt|W S  ty } zw|d}|r|d}|d}|r|r||krdd | D }|	d|i t
|}td}	|  	|	|	jt| |	jt|k}
t|
|  \}}|  }||| W d    n1 sw   Y  |W  Y d }~S |d }~ww )Nz1Unable to decode configuration from JSON string: hnsw_configuration
batch_sizesync_thresholdc                 S   s   i | ]\}}|d vr||qS ))r   r   r6   )r   r   r   r6   r6   r7   
<dictcomp>  s
    zCSqlSysDB._load_config_from_json_str_and_migrate.<locals>.<dictcomp>r}   )jsonloadsJSONDecodeErrorr   r   from_json_strr   getr   r   	from_jsonr   rC   r   r   r   r   rK   r<   r   rL   rB   rM   )r3   rx   r   config_jsonerrorr   r   r   rs   r   rX   rQ   rR   rO   r6   r6   r7   r     sN   



	
z/SqlSysDB._load_config_from_json_str_and_migratec                 C   s   ddl m} td}||pi }t|}ttd|dgd}| }| 	 
||jt||jt|k}	t|	|  \}
}|  }||
| W d   |S 1 sYw   Y  |S )zuInsert the configuration from legacy metadata params into the collections table, and return the configuration object.r   )PersistentHnswParamsr}   r   )r=   rn   )
parametersN)(chromadb.segment.impl.vector.hnsw_paramsr   r   extractr   from_legacy_paramsr   r   r   rC   r   r   r   r   rK   r<   r   rL   rB   rM   )r3   rx   rl   r   r   hnsw_metadata_paramsr   rs   configuration_json_strrX   rQ   rR   rO   r6   r6   r7   r   3  s8   	

z*SqlSysDB._insert_config_from_legacy_params)r8   N)NNNNr.   )5__name__
__module____qualname__r!   __annotations__r   r0   r    r   ALLr   r9   r   r
   rm   rT   r"   rZ   r^   r(   r`   r$   rr   r   r   r   r%   r   r   r   r&   r   r'   r   r   r   r   r   r)   r#   r*   r   r   r   r   r   r   r   r	   ro   r   r   __classcell__r6   r6   r4   r7   r+   ,   sZ  
 

,	


`
F
r

&


5


=

W
2r+   )7r   typingr   r   r   r   r   r   r   r	   uuidr
   	overridesr   pypikar   r   	itertoolsr   chromadb.api.configurationr   r   r   r   chromadb.configr   r   r   chromadb.db.baser   r   r   r   r   r   chromadb.db.systemr    chromadb.telemetry.opentelemetryr   r   r   r    chromadb.ingestr!   chromadb.typesr"   r#   r$   r%   r&   r'   r(   r)   r*   r+   r6   r6   r6   r7   <module>   s    ( ,