o
    3Ih                     @   sX  d dl Z d dlZd dlmZmZmZmZmZmZm	Z	m
Z
 d dlmZ d dlmZ d dlmZmZ d dlmZ d dlmZmZmZ d dlmZmZmZmZ d d	lmZ d d
lm Z m!Z! d dl"m#Z#m$Z$m%Z%m&Z& d dl'm(Z( d dl)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3 d dl4m5Z5m6Z6m7Z7m8Z8m9Z9m:Z:m;Z;m<Z<m=Z=m>Z> e ?e@ZAG dd deeZBdS )    N)OptionalSequenceAnyTuplecastDictUnionSet)UUID)override)TableColumn)groupby)DEFAULT_DATABASEDEFAULT_TENANTSystem)CursorSqlDBParameterValueget_sql)SysDB)NotFoundErrorUniqueConstraintError)add_attributes_to_current_spanOpenTelemetryClientOpenTelemetryGranularitytrace_method)Producer)
CollectionAndSegmentsDatabaseOptionalArgumentSegmentMetadata
CollectionSegmentScopeTenantUnspecifiedUpdateMetadata)
CreateCollectionConfigurationUpdateCollectionConfiguration+create_collection_configuration_to_json_str+load_collection_configuration_from_json_strCollectionConfiguration'create_collection_configuration_to_json collection_configuration_to_json$collection_configuration_to_json_str"overwrite_collection_configuration;update_collection_configuration_from_legacy_update_metadatac                       s2  e Zd ZU eed< def fddZedej	e
dO fdd	Ze
efd
edededdfddZe
efdededefddZe
efdededdfddZe
ddefdee dee dedee fddZe
deddfddZe
dedefddZdededdfddZedej	e
deddfddZed ej	e
ddd!eefd
eded"ed#ee d$ee  d%ee d&e!ded'ede"e#e!f fd(d Z$ed)ej	e
			dPd*ed
ee d+ee d,ee% dee f
d-d)Z&ed.ej	e
ddeeddfd
ee dee ded'edee dee dee# fd/d.Z'e
d0ede(fd1d2Z)ed3ej	e
d*ed
eddfd4d3Z*ded*eddfd5d6Z+ed7ej	e
eefd
eded'eddfd8d7Z,ed9ej	e
e- fd*ed
ed$e.ee/  ddfd:d9Z0ed;ej	e
e- e- e- e- fd
ede.e d%e.ee  d$e.ee/  d"e.ee1  ddfd<d;Z2ded=e1d
eddfd>d?Z3ed@ej	dAee"e4dBf  dee  fdCd@Z5edDej		dQdedEe6dFe7d
ed$e/dGee8e  ddfdHdDZ9d0e4d$ee  de:fdIdJZ;e
d
edefdKdLZ<e
edfded'ee defdMdNZ=  Z>S )RSqlSysDB	_producersystemc                    s   t  | |t| _d S N)super__init__requirer   _opentelemetry_client)selfr4   	__class__ X/home/air/sanwanet/gpt-api/venv/lib/python3.10/site-packages/chromadb/db/mixins/sysdb.pyr7   8   s   zSqlSysDB.__init__zSqlSysDB.create_segmentreturnNc                    s   t    | jt| _d S r5   )r6   start_systeminstancer   r3   )r:   r;   r=   r>   r@   <   s   
zSqlSysDB.startidnametenantc                 C   s   |   h}td}td}|  ||j|j|jt	| 
|t	||  |j||jt	|k}t||  \}}	z|||	 W n |  yc }
 ztd| d| |
d }
~
ww W d    d S 1 sow   Y  d S )N	databasestenants	Database z already exists for tenant )txr   querybuilderintocolumnsrC   rD   	tenant_idinsertr   
uuid_to_dbselectfrom_wherer   parameter_formatexecuteunique_constraint_errorr   )r:   rC   rD   rE   currF   rG   insert_databasesqlparamser=   r=   r>   create_databaseB   s8   
"zSqlSysDB.create_databasec           
      C   s   |   k}td}|  ||j|j|jt|k|j	t|k}t
||  \}}||| }|sDtd| d| d|d d u rUtd| d| dtt| |d }	t|	|d |dW  d    S 1 srw   Y  d S )NrF   rH    not found for tenant z. Are you sure it exists?r      rC   rD   rE   )rI   r   rJ   rQ   rP   rC   rD   rR   r   rM   r   rS   rT   fetchoner   r   r
   uuid_from_dbr   )
r:   rD   rE   rV   rF   qrX   rY   rowrC   r=   r=   r>   get_database_   s2   
$zSqlSysDB.get_databasec           
      C   s   |   m}td}|  ||jt|k|jt|k }t	|| 
 \}}|d }||| }|sCtd| d| td}	|  |	|	jt|d k }t	|| 
 \}}||| W d    d S 1 stw   Y  d S )NrF    RETURNING idrH   r\   collectionsr   )rI   r   rJ   rQ   rR   rD   r   rM   deleter   rS   rT   r_   r   database_id)
r:   rD   rE   rV   rF   ra   rX   rY   resultre   r=   r=   r>   delete_database{   s,   
"zSqlSysDB.delete_databaselimitoffsetc           
         s      M}td}  ||j|j|jt	k
||d u r(tjn||j}t|  \}}||| }	 fdd|	D W  d    S 1 sTw   Y  d S )NrF   c              	      s.   g | ]}t tt |d  |d dqS )r   r]   r^   )r   r   r
   r`   ).0rb   r:   rE   r=   r>   
<listcomp>   s    z+SqlSysDB.list_databases.<locals>.<listcomp>)rI   r   rJ   rQ   rP   rC   rD   rR   rM   r   rk   rj   sysmaxsizeorderby
created_atr   rS   rT   fetchall)
r:   rj   rk   rE   rV   rF   ra   rX   rY   rowsr=   rm   r>   list_databases   s$   
$zSqlSysDB.list_databasesc                 C   s   |   F}td}|  ||jt|}t|| 	 \}}z|
|| W n |  yA } z	td| d|d }~ww W d    d S 1 sMw   Y  d S )NrG   Tenant  already exists)rI   r   rJ   rK   rL   rC   rN   r   r   rS   rT   rU   r   )r:   rD   rV   rG   insert_tenantrX   rY   rZ   r=   r=   r>   create_tenant   s    

"zSqlSysDB.create_tenantc                 C   s   |   ?}td}|  ||j|jt|k}t|| 	 \}}|
|| }|s7td| dt|dW  d    S 1 sFw   Y  d S )NrG   rv   
 not found)rD   )rI   r   rJ   rQ   rP   rC   rR   r   r   rS   rT   r_   r   r%   )r:   rD   rV   rG   ra   rX   rY   rb   r=   r=   r>   
get_tenant   s   
$zSqlSysDB.get_tenantrV   segmentc           	   
   C   sH  t t|d |d |d jt|d d td}|  ||j|j|j	|j
t| |d t|d t|d jt| |d }t||  \}}z||| W n |  yr } ztd|d  d|d }~ww td	}|d
 rz| |||j|d |d
  W d S  ty } z	td|   d }~ww d S )NrC   typescope
collection
segment_idsegment_typesegment_scoper   segmentsSegment rw   segment_metadatametadataz"Error inserting segment metadata: )r   strvaluer   rJ   rK   rL   rC   r}   r~   r   rN   r   rO   r   rS   rT   rU   r   _insert_metadatar   	Exceptionloggererror)	r:   rV   r|   r   insert_segmentrX   rY   rZ   
metadata_tr=   r=   r>   create_segment_with_tx   sd   

	

zSqlSysDB.create_segment_with_txc                 C   s:   |   }| || W d    d S 1 sw   Y  d S r5   )rI   r   )r:   r|   rV   r=   r=   r>   create_segment  s   
"zSqlSysDB.create_collectionFconfigurationr   r   	dimensionget_or_createdatabasec
                 C   s  |d u r
|s
t dtt||d | j|||	d}
|
r8|r0|
d }| j|j||	dd dfS td| dt||t|||||	dd	}|  }t	d
}t	d}| 
 ||j|j|j|j|jt| |d t|d tt|t|d | 
 |j||jt|	k|jt|k}t||  \}}z||| W n |  y } ztd|d  d|d }~ww t	d}|d r| |||j|j|d  |D ]}| || qW d    |dfS 1 sw   Y  |dfS )Nz.id must be specified if get_or_create is Falsecollection_idcollection_name)rD   rE   r   r   )rC   rE   r   FCollection rw   rC   rD   configuration_jsonr   r   rE   r   versionre   rF   rC   rD   r   collection_metadatar   T) 
ValueErrorr   r   get_collectionsrC   r   r#   r-   rI   r   rJ   rK   rL   rD   config_json_strr   rg   rN   r   rO   r*   rP   rQ   rR   rM   r   rS   rT   rU   r   r   r   )r:   rC   rD   r   r   r   r   r   rE   r   existingr   rV   re   rF   insert_collectionrX   rY   rZ   r   r|   r=   r=   r>   create_collection  s   



11zSqlSysDB.get_segmentsr   r}   r~   c                 C   s  t t||r|nd|r|jndt|d td}td}|  ||j|j|j	|j
|j|j|j|j|j	||j|jk|j}|rV||jt| |k}|rb||jt|k}|ro||j	t|jk}|r~||j
t| |k}|  f}t||  \}	}
||	|
 }t|dd }g }|D ]?\}}| t|}t|}t|d d }tt|d d	 }| |d d
 }|  |}|!t"t#t$|||||i d q|W  d    S 1 sw   Y  d S )N r   r   r   c                 S      t t| d S Nr   r   objectrr=   r=   r>   <lambda>      z'SqlSysDB.get_segments.<locals>.<lambda>r   r]         )rC   r}   r~   r   r   
file_paths)%r   r   r   r   rJ   rQ   rP   rC   r}   r~   r   key	str_value	int_valuefloat_value
bool_value	left_joinonr   rq   rR   r   rO   rI   r   rS   rT   rs   r   r`   listr$   _metadata_from_rowsappendr!   r   r
   )r:   r   rC   r}   r~   
segments_tr   ra   rV   rX   rY   rt   
by_segmentr   r   segment_rowsr   r=   r=   r>   get_segmentsu  sx   	


$zSqlSysDB.get_collectionsc                 C   s  |dur|du s|du rt dtt||r|ndd td}td}td}	|  ||j|j|j	|j
|	j|	j|j|j|j|j|j||j|jk|	|j|	jk|j}
|ro|
|jt| |k}
|r{|
|jt|k}
|du r|r|rtd}	|
|j|  |	j|	|	jt|k|	jt|kk}
|  }t|
|  \}}||| }t|dd	 }g }|D ]d\}}| t|}t |}t|d
 d }| !|}|d
 d rt"|d
 d nd}|d
 d durt#|d
 d }n| $||}|%t&t't(||t)|||t|d
 d t|d
 d d
d q|durC|du r:d
}||||  }n||d }|W  d   S 1 sVw   Y  dS )z;Get collections by name, embedding function and/or metadataNzmIf name is specified, tenant and database must also be specified in order to uniquely identify the collectionr   r   re   r   rF   c                 S   r   r   r   r   r=   r=   r>   r     r   z*SqlSysDB.get_collections.<locals>.<lambda>r   r]   r   r         r   )*r   r   r   r   rJ   rQ   rP   rC   rD   r   r   rM   r   r   r   r   r   r   r   r   rg   rq   rR   r   rO   rI   r   rS   rT   rs   r   r`   r   r   intr+   !_insert_config_from_legacy_paramsr   r#   r   r
   r.   )r:   rC   rD   rE   r   rj   rk   collections_tr   databases_tra   rV   rX   rY   rt   by_collectionre   r   collection_rowsr   r   r   r=   r=   r>   r     s   



 


&r   c                 C   s@   | j |d}t|dkrtd| dt|d | j|ddS )N)rC   r   r   z does not exist.)r   )r   r   )r   lenr   r   r   )r:   r   re   r=   r=   r>   get_collection_with_segments4  s   
z%SqlSysDB.get_collection_with_segmentszSqlSysDB.delete_segmentc           	      C   s   t dt|i td}|  ||jt| |k	 }| 
 (}t||  \}}|d }||| }|sDtd| dW d   dS 1 sOw   Y  dS )zDelete a segment from the SysDBr   r   rd   r   rz   N)r   r   r   rJ   rQ   rR   rC   r   rO   rf   rI   r   rS   rT   r_   r   )	r:   r   rC   tra   rV   rX   rY   rh   r=   r=   r>   delete_segment@  s$   
"c                 C   sR   t d}|  ||jt| |k }t|| 	 \}}|
|| d S )Nr   )r   rJ   rQ   rR   r   r   rO   rf   r   rS   rT   )r:   rV   r   r   ra   rX   rY   r=   r=   r>   delete_segments_for_collectionZ  s   z'SqlSysDB.delete_segments_for_collectionzSqlSysDB.delete_collectionc                 C   s  t dt|i td}td}|  ||jt| |k|j	|  
|j||jt|k|jt|kk }|  -}t||  \}}	|d }|||	 }
|
sgtd| d| || W d   n1 sww   Y  | j|
d  dS )	z{Delete a collection and all associated segments from the SysDB. Deletes
        the log stream for this collection as well.r   re   rF   rd   r   rz   Nr   )r   r   r   rJ   rQ   rR   rC   r   rO   rg   rP   rD   rM   rf   rI   r   rS   rT   r_   r   r   r3   
delete_log)r:   rC   rE   r   r   r   ra   rV   rX   rY   rh   r=   r=   r>   delete_collectione  s:   


zSqlSysDB.update_segmentc           
      C   sZ  t t|t|d td}td}|  ||jt| |k	|j
t| |}|  p}t||  \}}	|rE|||	 |d u rm|  ||jt| |k }t||  \}}	|||	 n.|t krtt|}tt|}| |||j||t	|  W d    d S W d    d S W d    d S 1 sw   Y  d S )N)r   r   r   r   )r   r   r   rJ   updaterR   rC   r   rO   setr   rI   r   rS   rT   rQ   r   rf   r&   r   r'   r   keys)
r:   r   rC   r   r   r   ra   rV   rX   rY   r=   r=   r>   update_segment  sP   




"zSqlSysDB.update_collectionc                 C   s  t dt|i td}td}|  ||jt| |k}|t	 ks0|
|jt|}|t	 ks>|
|jt|}|  }	t||  \}
}|
rd|
d }
|	|
|}| sdtd| d|t	 kr|  ||jt| |k }t||  \}
}|	|
| |d urtt|}| |	||j||t
|  |t	 krtt|}| |	|| n)|t	 krtt|}|d urt|}| |	|| W d    d S W d    d S W d    d S W d    d S 1 sw   Y  d S )Nr   re   r   rd   r   rz   )r   r   r   rJ   r   rR   rC   r   rO   r&   r   rD   r   rI   r   rS   rT   r_   r   rQ   r   rf   r   r'   r   r   r)   _update_config_json_strr1   )r:   rC   rD   r   r   r   r   r   ra   rV   rX   rY   rh   update_configurationr=   r=   r>   update_collection  sz   






	

(&*"r   c                 C   s   t d}|  ||j|jt| |k}t	|| 
 \}}||| }|s5td| d|d }	t|	}
t|
|}|  ||jtt||jt| |k}t	|| 
 \}}||| d S )Nre   r   rz   r   )r   rJ   rQ   rP   r   rR   rC   r   rO   r   rS   rT   r_   r   r+   r0   r   r   r/   )r:   rV   r   rC   r   ra   rX   rY   rb   r   existing_config
new_configr=   r=   r>   r     s2   
	z SqlSysDB._update_config_json_strzSqlSysDB._metadata_from_rowsrt   .c                 C   s   t dt|i i }|D ]C}t|d }|d dur#t|d ||< q|d dur2t|d ||< q|d durAt|d ||< q|d durOt|d ||< q|pSdS )zGiven SQL rows, return a metadata map (assuming that the last four columns
        are the key, str_value, int_value & float_value)num_rowsN)r   r   r   r   floatbool)r:   rt   r   rb   r   r=   r=   r>   r   3  s"   zSqlSysDB._insert_metadatatableid_col
clear_keysc                 C   s  t dt|i |r8|  ||t| |k|jdd |D 	 }t
||  \}}	|||	 |  |||j|j|j|j|j}| |}
| D ]c\}}t|trq|t|
t|d d d tt|}qUt|tr|t|
t|t|d d d }qUt|tr|t|
t|d t|d d }qUt|tr|t|
t|d d t|d }qU|d u rqUqUt
||  \}}	|r|||	 d S d S )Nnum_keysc                 S   s   g | ]}t |qS r=   )r   )rl   kr=   r=   r>   rn   b  s    z-SqlSysDB._insert_metadata.<locals>.<listcomp>)r   r   rJ   rQ   rR   r   rO   r   isinrf   r   rS   rT   rK   rL   r   r   r   r   items
isinstancer   rN   r   r   r   )r:   rV   r   r   rC   r   r   ra   rX   rY   sql_idr   vr=   r=   r>   r   K  s   





c           
      C   s   t d}t }t|}|  ||jt||j	t|k}t
||  \}}|  }	|	|| W d   t|S 1 sBw   Y  t|S )zuInsert the configuration from legacy metadata params into the collections table, and return the configuration object.re   N)r   r(   r*   rJ   r   r   r   r   rR   rC   r   rS   rI   rT   r+   )
r:   r   r   r   create_collection_configconfiguration_json_strra   rX   rY   rV   r=   r=   r>   r     s(   	

z*SqlSysDB._insert_config_from_legacy_paramsc                 C   s   t r5   )NotImplementedError)r:   rC   r=   r=   r>   get_collection_size  s   zSqlSysDB.get_collection_sizec                 C   s*   |du s|dkr
dn|}t | j||dS )zFGets the number of collections for the (tenant, database) combination.Nr   )rE   r   )r   r   )r:   rE   r   request_databaser=   r=   r>   count_collections  s   zSqlSysDB.count_collections)r?   N)NNNr5   )?__name__
__module____qualname__r   __annotations__r   r7   r   r   ALLr   r@   r   r
   r   r[   r   rc   ri   r   r   r   ru   ry   r%   r{   r   r!   r   r   r   r(   r"   r   r   r#   r   r$   r   r   r   r   r   r   r   r&   r    r'   r   r)   r   r   r   r   r   r   r	   r   r,   r   r   r   __classcell__r=   r=   r;   r>   r2   3   s  
 

3
	


a
Gt


)

0

	P



W
r2   )Cloggingro   typingr   r   r   r   r   r   r   r	   uuidr
   	overridesr   pypikar   r   	itertoolsr   chromadb.configr   r   r   chromadb.db.baser   r   r   r   chromadb.db.systemr   chromadb.errorsr   r    chromadb.telemetry.opentelemetryr   r   r   r   chromadb.ingestr   chromadb.typesr   r   r    r!   r"   r#   r$   r%   r&   r'   %chromadb.api.collection_configurationr(   r)   r*   r+   r,   r-   r.   r/   r0   r1   	getLoggerr   r   r2   r=   r=   r=   r>   <module>   s"    (00
