o
    3Ihd                     @   s  d dl mZmZmZmZmZmZmZmZm	Z	 d dl
mZ d dlmZ d dlmZ d dlmZmZmZ d dlmZ d dlmZ d dlmZmZmZ d d	lmZmZmZ d d
lm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z( d dl)m*Z* d dl+m,Z,m-Z- d dl.m/Z/ d dl0m1Z2 d dl3m4Z4 d dl5m6Z6 d dl7m8Z8 d dl9Z9d dl:Z:e:;e<Z=G dd deZ>de?dee'ee(e'f eee	e' f f de/de,de,de4fddZ@de?dee'e	e' f dee(ef de/de,de,de4fddZAdS )     )	OptionalSequenceAnyTuplecast	GeneratorUnionDictList)MetadataReader)Consumer)System)RequestVersionContextSegmentInclusionExclusionOperator)SqliteDB)override)CursorParameterValueget_sql)OpenTelemetryClientOpenTelemetryGranularitytrace_method)	WhereWhereDocumentMetadataEmbeddingRecord	LogRecordSeqId	OperationUpdateMetadataLiteralValueWhereOperator)UUID)TableTables)QueryBuilderN)	Criterion)groupby)reducec                   @   s  e Zd ZU eed< eed< eed< eed< ee ed< dZ	ee ed< de
d	efd
dZedejedAddZedejedAddZedejedefddZedejededefddZedeje						dBdedee dee deee  dee dee dedee fddZded e dede!eddf fd!d"Z"ed#ejd$ee#e$d%f  dedefd&d#Z%ed'ejded(e&d)eddfd*d'Z'ed+ejded,ed-e(ddfd.d+Z)ed/ejded,ed-e(ddfd0d/Z*ed1ejded(e&ddfd2d1Z+ed3ejded(e&ddfd4d3Z,ed5ejd6ee& ddfd7d5Z-ed8ejd e ded9e.d:e.de/f
d;d8Z0ed<ejd e ded9e.d=e.d:e.de/fd>d<Z1ed?ejedAd@d?Z2dS )CSqliteMetadataSegment	_consumer_db_id_opentelemetry_client_collection_idN_subscriptionsystemsegmentc                 C   s<   | t| _| t| _|d | _|t| _|d | _	d S )Nid
collection)
instancer   r+   r   r*   r,   requirer   r-   r.   )selfr0   r1    r7   e/home/air/sanwanet/gpt-api/venv/lib/python3.10/site-packages/chromadb/segment/impl/metadata/sqlite.py__init__3   s
   
zSqliteMetadataSegment.__init__zSqliteMetadataSegment.startreturnc                 C   s.   | j r|  }| jj| j | j|d| _d S d S )N)collection_id
consume_fnstart)r.   	max_seqidr*   	subscribe_write_metadatar/   )r6   seq_idr7   r7   r8   r=   :   s   zSqliteMetadataSegment.stopc                 C   s   | j r| j| j  d S d S N)r/   r*   unsubscribe)r6   r7   r7   r8   stopE   s   zSqliteMetadataSegment.max_seqidc                 C   s   t d}| j ||j|jt| j	| j
k}t|\}}| j '}||| }|d u rA| j W  d    S tt|d W  d    S 1 sRw   Y  d S )N
max_seq_idr   )r#   r+   querybuilderfrom_selectrA   where
segment_idr   
uuid_to_dbr,   r   txexecutefetchoner*   	min_seqidr   int)r6   tqsqlparamscurresultr7   r7   r8   r>   K   s   $zSqliteMetadataSegment.countrequest_version_contextc                 C   s   t d}| j ||jt| j| jk	t
|j}t|\}}| j }||| d }tt|W  d    S 1 sEw   Y  d S )N
embeddingsr   )r#   r+   rF   rG   rI   rJ   r   rK   r,   rH   fnCountr2   r   rL   rM   rN   r   rP   )r6   rW   embeddings_trR   rS   rT   rU   rV   r7   r7   r8   count^   s   $z"SqliteMetadataSegment.get_metadataTrI   where_documentidslimitoffsetinclude_metadatac              	   C   s,  t ddd\}}	}
|pd}|pd}|dk rtd|j|j|jg}|r2||	j|	j|	j|	j	|	j
g | j ||	|j|	jkj| |j}|dusS|dur| j ||j|	|j|	jk|j|jt| j| jk }|r|| |||	|}|r|| |||	|
|}|dur||jt|}||}||}||j|}n;| j ||j|jt| j| jk|j||}|dur||jt|}||j|}| j }t| |||W  d   S 1 sw   Y  dS )zQuery for embedding metadata.rX   embedding_metadataembedding_fulltext_searchl    r   zLimit cannot be negativeN) r$   
ValueErrorr2   embedding_idrA   extendkeystring_value	int_valuefloat_value
bool_valuer+   rF   rG   	left_joinonrH   orderbyrI   rJ   r   rK   r,   distinct_where_map_criterion_where_doc_criterionisinr_   r`   rL   list_records)r6   rW   rI   r]   r^   r_   r`   ra   r[   
metadata_t
fulltext_tselect_clauserR   
metadata_qembeddings_qrU   r7   r7   r8   get_metadatao   s   



&rU   rR   c           
      c   sX    t |\}}||| t|jd}t|dd }|D ]\}}	| t|	|V  qdS )zqGiven a cursor and a QueryBuilder, yield a generator of records. Assumes
        cursor returns rows in ID order.Nc                 S   s   t | d S )Nr   )rP   )rr7   r7   r8   <lambda>   s    z0SqliteMetadataSegment._records.<locals>.<lambda>)r   rM   iterrN   r'   _recordrs   )
r6   rU   rR   ra   rS   rT   cur_iteratorgroup_iterator_groupr7   r7   r8   rt      s   zSqliteMetadataSegment._recordszSqliteMetadataSegment._recordrows.c                 C   s   |d dd \}}}|st |ddS i }|D ]9}|dd \}}	}
}}|	dur-|	||< q|
dur6|
||< q|dur?|||< q|durP|dkrLd||< qd||< qt ||pVddS )zUGiven a list of DB rows with the same ID, construct a
        MetadataEmbeddingRecordr   N   )r2   metadata   TF)r   )r6   r   ra   r   re   rA   r   rowrg   rh   ri   rj   rk   r7   r7   r8   r~      s*   



z$SqliteMetadataSegment._insert_recordrecordupsertc           	      C   s   t d}| j ||j|j|j|jt	| j
| jk|jt	|d d kt	| j
| jt	|d d t	|d }t|\}}|d }z||| d }W n! tjyz   |rk| || Y S td|d d   Y dS w |d d	 r| |||d d	  dS dS )
z2Add or update a single EmbeddingRecord into the DBrX   r   r2   
log_offsetzRETURNING idr   z!Insert of existing embedding ID: Nr   )r#   r+   rF   intocolumnsrJ   re   rA   rI   r   rK   r,   insertr   rM   rN   sqlite3IntegrityError_update_recordloggerwarning_update_metadata)	r6   rU   r   r   rQ   rR   rS   rT   r2   r7   r7   r8   _insert_record  s6   
z&SqliteMetadataSegment._update_metadatar2   r   c           	      C   s~   t d}dd | D }|r6| j ||jt|k|j	t|
 }t|\}}||| | ||| dS )z0Update the metadata for a single EmbeddingRecordrb   c                 S   s   g | ]
\}}|d u r|qS rB   r7   ).0kvr7   r7   r8   
<listcomp>9  s    z:SqliteMetadataSegment._update_metadata.<locals>.<listcomp>N)r#   itemsr+   rF   rG   rI   r2   r   rg   rr   deleter   rM   _insert_metadata)	r6   rU   r2   r   rQ   	to_deleterR   rS   rT   r7   r7   r8   r   3  s   z&SqliteMetadataSegment._insert_metadatac           
   
      s  t dj jjjjj	j
} D ][\}}t|tr8|tt|t|ddd}qt|trN|tt|dddt|}qt|trd|tt|dt|dd}qt|try|tt|ddt|d}qt|\}}|dd}|r || dv rt dd
 fdd	}	z|	  W dS  tjy   j jtk }t|\}} || |	  Y dS w dS )z@Insert or update each metadata row for a single embedding recordrb   NINSERTINSERT OR REPLACEchroma:documentrc   r:   c                     sL   j  jjttd } t| \}} 	|| d S )Nr   )
r+   rF   r   r   rowidrh   r   r   r   rM   )rR   rS   rT   rU   r2   r   r6   rQ   r7   r8   insert_into_fulltext_search  s   
	zKSqliteMetadataSegment._insert_metadata.<locals>.insert_into_fulltext_searchr:   N)r#   r+   rF   r   r   r2   rg   rh   ri   rj   rk   r   
isinstancestrr   r   boolrP   floatr   replacerM   r   r   rG   rI   r   r   )
r6   rU   r2   r   rR   rg   valuerS   rT   r   r7   r   r8   r   G  s   

	

	z$SqliteMetadataSegment._delete_recordc                 C   sd  t d}t d}| j ||jt| j| jk|j	t|d d k
 }| j |
 |j| j ||j|jt| j| jk|j	t|d d k}|jt|  t|\}}|d }||| }	|	du rtd|d d   dS |	d }
t d	}| j ||jt|
k
 }t|\}}||| dS )
z+Delete a single EmbeddingRecord from the DBrX   rc   r   r2    RETURNING idNz$Delete of nonexisting embedding ID: r   rb   )r#   r+   rF   rG   rI   rJ   r   rK   r,   re   r   r   rr   rH   r2   rM   r   rN   r   r   )r6   rU   r   rQ   fts_trR   q_ftsrS   rT   rV   r2   ru   r7   r7   r8   _delete_record  sP   z$SqliteMetadataSegment._update_recordc           	      C   s   t d}| j ||jt|d |jt| j	| j
k|jt|d d k}t|\}}|d }||| }|du rRtd|d d   dS |d }|d d	 ri| |||d d	  dS dS )
z)Update a single EmbeddingRecord in the DBrX   r   r   r2   r   Nz$Update of nonexisting embedding ID: r   r   )r#   r+   rF   updatesetrA   r   rI   rJ   rK   r,   re   r   rM   rN   r   r   r   )	r6   rU   r   rQ   rR   rS   rT   rV   r2   r7   r7   r8   r     s$   z%SqliteMetadataSegment._write_metadatarecordsc                 C   s  | j  ~}|D ]C}|d d tjkr| ||d q|d d tjkr,| ||d q|d d tjkr<| || q|d d tjkrK| 	|| q| j 
 tdddt| j | jt|d }t|\}}|d	d
}||| W d   dS 1 sw   Y  dS )zWrite embedding metadata to the database. Care should be taken to ensure
        records are append-only (that is, that seq-ids should increase monotonically)r   	operationFTrE   rJ   rA   r   r   r   N)r+   rL   r   ADDr   UPSERTDELETEr   UPDATEr   rF   r   r#   r   r   r   rK   r,   r   r   rM   )r6   r   rU   r   rR   rS   rT   r7   r7   r8   r@     s.   

	"z*SqliteMetadataSegment._where_map_criterionru   r[   c           
   	      s   g }|  D ]\\}}|dkr* fddttt |D }|tdd | q|dkrJ fddttt |D }|tdd | qttttt	tf f |}	|t
||	  qtd	d |S )
N$andc                       g | ]
} | qS r7   rp   r   wr[   ru   rR   r6   r7   r8   r         z>SqliteMetadataSegment._where_map_criterion.<locals>.<listcomp>c                 S      | |@ S rB   r7   xyr7   r7   r8   r|         z<SqliteMetadataSegment._where_map_criterion.<locals>.<lambda>$orc                    r   r7   r   r   r   r7   r8   r     r   c                 S      | |B S rB   r7   r   r7   r7   r8   r|     r   c                 S   r   rB   r7   r   r7   r7   r8   r|      r   )r   r   r   r   appendr(   r   r    r	   r!   _where_clause)
r6   rR   rI   ru   r[   clauser   r   criteriaexprr7   r   r8   rp   	  s   z*SqliteMetadataSegment._where_doc_criterionrv   c                    s  |  D ]\}}|dkr' fddttt |D }tdd |  S |dkrF fddttt |D }tdd |  S |d	v r~tt|}d
| d
}	j 	j
jt|	}
|dkrv j|
  S  j|
  S td| td)Nr   c              	         g | ]} | qS r7   rq   r   r[   rv   ru   rR   r6   r7   r8   r   /      
z>SqliteMetadataSegment._where_doc_criterion.<locals>.<listcomp>c                 S   r   rB   r7   r   r7   r7   r8   r|   5  r   z<SqliteMetadataSegment._where_doc_criterion.<locals>.<lambda>r   c              	      r   r7   r   r   r   r7   r8   r   7  r   c                 S   r   rB   r7   r   r7   r7   r8   r|   =  r   )	$containsz$not_contains%r   zUnknown where_doc operator zEmpty where_doc)r   r   r   r   r(   r   r+   rF   rG   rH   r   rI   rh   liker   r2   rr   notinrd   )r6   rR   rI   ru   rv   r[   r   r   r   search_termsqr7   r   r8   rq   "  s4   


zSqliteMetadataSegment.deletec                 C   sb  t d}t d}t d}| j | |j| j ||j|j	t
| j| jk}| j | |j| j ||j|j	t
| j| jk}| j | |j| j ||j|j	t
| j| jk}| j }|jt|  |jt|  |jt|  W d    d S 1 sw   Y  d S )NrX   rb   rc   )r#   r+   rF   rG   r   rI   r2   rr   rH   rJ   r   rK   r,   r   rL   rM   r   )r6   rQ   t1t2q0rR   r   rU   r7   r7   r8   r   Q  sd   "r   )NNNNNT)3__name__
__module____qualname__r   __annotations__r   r"   r   r   r/   r   r   r9   r   r   ALLr   r=   rD   r   r>   r   rP   r\   r   r   r   r   r   r   rz   r   r%   r   rt   r   r   r~   r   r   r   r   r   r   r   r@   r#   r&   rp   rq   r   r7   r7   r7   r8   r)   +   s   
 

	



	r


"
X
3

,r)   rg   r   rx   ru   r[   r:   c                 C   sR   t |ttttfrt| ttd|i|||S tt	|
 \}}t| |||||S )zLGiven a field name, an expression, and a table, construct a Pypika Criterion$eq)r   r   rP   r   r   r   r   r!   nextr}   r   _value_criterion)rg   r   rx   ru   r[   operatorr   r7   r7   r8   r     s   	r   r   opc                 C   sj  dt dtfdd}||jt| k}t|}||s&t|try||d ry|j|j}	}
|dv r:|	|k|
|kB }ne|dkrG|	|k|
|kB }nX|dkrT|	|k|
|kB }nK|d	kra|	|k |
|k B }n>|d
krn|	|k|
|kB }n1|		||
	|B }n&t|tst|trt|d tr|j
}n|j}|dv r||k}n|	|}|dv r|j||S |j	||S )z(Creates the filter for a single operatorobjr:   c                 S   s   t | t ot | ttfS rB   )r   r   rP   r   )r   r7   r7   r8   
is_numeric  s   z$_value_criterion.<locals>.is_numericr   )r   $nez$gtz$gtez$ltz$lte)r   z$nin)objectr   rI   rg   r   r   rs   ri   rj   rr   rk   rh   r2   r   )rg   r   r   rx   ru   r[   r   sub_qp_valint_col	float_colr   colr7   r7   r8   r     s:   



r   )Btypingr   r   r   r   r   r   r   r	   r
   chromadb.segmentr   chromadb.ingestr   chromadb.configr   chromadb.typesr   r   r   chromadb.db.impl.sqliter   	overridesr   chromadb.db.baser   r   r    chromadb.telemetry.opentelemetryr   r   r   r   r   r   r   r   r   r   r    r!   uuidr"   pypikar#   r$   pypika.queriesr%   pypika.functions	functionsrY   pypika.termsr&   	itertoolsr'   	functoolsr(   r   logging	getLoggerr   r   r)   r   r   r   r7   r7   r7   r8   <module>   st   , ,
    c
	


