o
    1Xxil                     @   s  d dl mZmZmZmZmZmZmZmZm	Z	 d dl
mZ d dlmZ d dlmZ d dlmZmZ d dlmZ d dlmZ d dlmZmZmZ d d	lmZmZmZ d d
lmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z' d dl(m)Z) d dl*m+Z+m,Z, d dl-m.Z. d dl/m0Z1 d dl2m3Z3 d dl4m5Z5 d dl6m7Z7 d dl8Z8d dl9Z9e9:e;Z<G dd deZ=de#de>fddZ?de>de#fddZ@dee&ee'e&f eee	e& f f de+de3fddZAdee&e	e& f d ee'ef de+de3fd!d"ZBdS )#    )	OptionalSequenceAnyTuplecast	GeneratorUnionDictList)MetadataReader)Consumer)System)SegmentInclusionExclusionOperator)SqliteDB)override)CursorParameterValueget_sql)OpenTelemetryClientOpenTelemetryGranularitytrace_method)	WhereWhereDocumentMetadataEmbeddingRecord	LogRecordSeqId	OperationUpdateMetadataLiteralValueWhereOperator)UUID)TableTables)QueryBuilderN)	Criterion)groupby)reducec                   @   s  e Zd ZU eed< eed< eed< eed< ee ed< ee ed< de	de
fd	d
Zedejed>ddZedejed>ddZedejedefddZedejedefddZedeje					d?dee dee deee  dee dee dee fddZdededeeddf fddZed ejd!ee e!d"f  defd#d Z"ed$ejded%e#d&e$ddfd'd$Z%ed(ejded)ed*e&ddfd+d(Z'ed,ejded)ed*e&ddfd-d,Z(ed.ejded%e#ddfd/d.Z)ed0ejded%e#ddfd1d0Z*ed2ejd3ee# ddfd4d2Z+ed5ejdeded6e,d7e,de-f
d8d5Z.ed9ejdeded6e,d:e,d7e,de-fd;d9Z/ed<ejed>d=d<Z0dS )@SqliteMetadataSegment	_consumer_db_id_opentelemetry_client_collection_id_subscriptionsystemsegmentc                 C   s<   | t| _| t| _|d | _|t| _|d | _	d S )Nid
collection)
instancer   r*   r   r)   r+   requirer   r,   r-   )selfr/   r0    r6   d/home/air/biblejyuku/back/venv/lib/python3.10/site-packages/chromadb/segment/impl/metadata/sqlite.py__init__3   s
   
zSqliteMetadataSegment.__init__zSqliteMetadataSegment.startreturnNc                 C   s.   | j r|  }| jj| j | j|d| _d S d S )N)collection_id
consume_fnstart)r-   	max_seqidr)   	subscribe_write_metadatar.   )r5   seq_idr6   r6   r7   r<   :   s   zSqliteMetadataSegment.stopc                 C   s   | j r| j| j  d S d S N)r.   r)   unsubscribe)r5   r6   r6   r7   stopE   s   zSqliteMetadataSegment.max_seqidc                 C   s   t d}| j ||j|jt| j	| j
k}t|\}}| j &}||| }|d u rA| j W  d    S t|d W  d    S 1 sQw   Y  d S )N
max_seq_idr   )r"   r*   querybuilderfrom_selectr@   where
segment_idr   
uuid_to_dbr+   r   txexecutefetchoner)   	min_seqid_decode_seq_id)r5   tqsqlparamscurresultr6   r6   r7   r=   K   s   
$zSqliteMetadataSegment.countc                 C   s   t d}| j ||jt| j| jk	t
|j}t|\}}| j }||| d }tt|W  d    S 1 sEw   Y  d S )N
embeddingsr   )r"   r*   rE   rF   rH   rI   r   rJ   r+   rG   fnCountr1   r   rK   rL   rM   r   int)r5   embeddings_trQ   rR   rS   rT   rU   r6   r6   r7   count^   s   $z"SqliteMetadataSegment.get_metadatarH   where_documentidslimitoffsetc              
   C   s  t ddd\}}}|pd}|pd}|dk rtd| j |||j|jk|j|j	|j
|j|j|j|j|j|j	}	|dusI|dur| j ||j||j|jk|j	|jt| j| jk }
|r|
| |
|||}
|r|
| |
||||}
|dur|
|j	t|}
|
|}
|
|}
|	|j|
}	n;| j ||j|jt| j| jk|j	||}|dur||j	t|}|	|j|}	| j }t| ||	W  d   S 1 sw   Y  dS )zQuery for embedding metadata.rV   embedding_metadataembedding_fulltext_searchl    r   zLimit cannot be negativeN) r#   
ValueErrorr*   rE   rF   	left_joinonr1   rG   embedding_idr@   keystring_value	int_valuefloat_value
bool_valueorderbyjoinrH   rI   r   rJ   r+   distinct_where_map_criterion_where_doc_criterionisinr^   r_   rK   list_records)r5   rH   r\   r]   r^   r_   rZ   
metadata_t
fulltext_trQ   
metadata_qembeddings_qrT   r6   r6   r7   get_metadatao   s   




&rT   rQ   c           	      c   sV    t |\}}||| t|jd}t|dd }|D ]\}}| t|V  qdS )zqGiven a cursor and a QueryBuilder, yield a generator of records. Assumes
        cursor returns rows in ID order.Nc                 S   s   t | d S )Nr   )rY   )rr6   r6   r7   <lambda>   s    z0SqliteMetadataSegment._records.<locals>.<lambda>)r   rL   iterrM   r&   _recordrq   )	r5   rT   rQ   rR   rS   cur_iteratorgroup_iterator_groupr6   r6   r7   rr      s   zSqliteMetadataSegment._recordszSqliteMetadataSegment._recordrows.c                 C   s   |d dd \}}}i }|D ]9}|dd \}}}	}
}|dur%|||< q|	dur.|	||< q|
dur7|
||< q|durH|dkrDd||< qd||< qt ||pNddS )zUGiven a list of DB rows with the same ID, construct a
        MetadataEmbeddingRecordr   N      TF)r1   metadata)r   )r5   r   r~   re   r@   r   rowrf   rg   rh   ri   rj   r6   r6   r7   r{      s&   



z$SqliteMetadataSegment._insert_recordrecordupsertc           	      C   s$  t d}| j ||j|j|j|jt	| j
| jk|jt	|d d kt	| j
| jt	|d d t	t|d }t|\}}|d }z||| d }W n! tjy|   |rm| || Y S td|d d   Y dS w |d d	 r| |||d d	  dS dS )
z2Add or update a single EmbeddingRecord into the DBrV   r   r1   
log_offsetzRETURNING idr   z!Insert of existing embedding ID: Nr   )r"   r*   rE   intocolumnsrI   re   r@   rH   r   rJ   r+   insert_encode_seq_idr   rL   rM   sqlite3IntegrityError_update_recordloggerwarning_update_metadata)	r5   rT   r   r   rP   rQ   rR   rS   r1   r6   r6   r7   _insert_record  s6   z&SqliteMetadataSegment._update_metadatar1   r   c           	      C   s~   t d}dd | D }|r6| j ||jt|k|j	t|
 }t|\}}||| | ||| dS )z0Update the metadata for a single EmbeddingRecordr`   c                 S   s   g | ]
\}}|d u r|qS rA   r6   ).0kvr6   r6   r7   
<listcomp>+  s    z:SqliteMetadataSegment._update_metadata.<locals>.<listcomp>N)r"   itemsr*   rE   rF   rH   r1   r   rf   rp   deleter   rL   _insert_metadata)	r5   rT   r1   r   rP   	to_deleterQ   rR   rS   r6   r6   r7   r   %  s   z&SqliteMetadataSegment._insert_metadatac           
   
      s  t dj jjjjj	j
} D ][\}}t|tr8|tt|t|ddd}qt|trN|tt|dddt|}qt|trd|tt|dt|dd}qt|try|tt|ddt|d}qt|\}}|dd}|r || dv rt dd
 fdd	}	z|	  W dS  tjy   j jtk }t|\}} || |	  Y dS w dS )z@Insert or update each metadata row for a single embedding recordr`   NINSERTINSERT OR REPLACEchroma:documentra   r9   c                     sL   j  jjttd } t| \}} 	|| d S )Nr   )
r*   rE   r   r   rowidrg   r   r   r   rL   )rQ   rR   rS   rT   r1   r   r5   rP   r6   r7   insert_into_fulltext_searchz  s   
	zKSqliteMetadataSegment._insert_metadata.<locals>.insert_into_fulltext_searchr9   N)r"   r*   rE   r   r   r1   rf   rg   rh   ri   rj   r   
isinstancestrr   r   boolrY   floatr   replacerL   r   r   rF   rH   r   r   )
r5   rT   r1   r   rQ   rf   valuerR   rS   r   r6   r   r7   r   9  s   

	

	z$SqliteMetadataSegment._delete_recordc                 C   sd  t d}t d}| j ||jt| j| jk|j	t|d d k
 }| j |
 |j| j ||j|jt| j| jk|j	t|d d k}|jt|  t|\}}|d }||| }	|	du rtd|d d   dS |	d }
t d	}| j ||jt|
k
 }t|\}}||| dS )
z+Delete a single EmbeddingRecord from the DBrV   ra   r   r1    RETURNING idNz$Delete of nonexisting embedding ID: r   r`   )r"   r*   rE   rF   rH   rI   r   rJ   r+   re   r   r   rp   rG   r1   rL   r   rM   r   r   )r5   rT   r   rP   fts_trQ   q_ftsrR   rS   rU   r1   rs   r6   r6   r7   _delete_record  sP   z$SqliteMetadataSegment._update_recordc           	      C   s   t d}| j ||jtt|d |j	t| j
| jk|jt|d d k}t|\}}|d }||| }|du rTtd|d d   dS |d }|d d	 rk| |||d d	  dS dS )
z)Update a single EmbeddingRecord in the DBrV   r   r   r1   r   Nz$Update of nonexisting embedding ID: r   r   )r"   r*   rE   updatesetr@   r   r   rH   rI   rJ   r+   re   r   rL   rM   r   r   r   )	r5   rT   r   rP   rQ   rR   rS   rU   r1   r6   r6   r7   r     s$   z%SqliteMetadataSegment._write_metadatarecordsc              	   C   s  | j  }|D ]t}| j  tdddt| j | j	tt
|d }t|\}}|dd}||| |d d tjkrL| ||d	 q|d d tjkr]| ||d
 q|d d tjkrm| || q|d d tjkr|| || qW d   dS 1 sw   Y  dS )zWrite embedding metadata to the database. Care should be taken to ensure
        records are append-only (that is, that seq-ids should increase monotonically)rD   rI   r@   r   r   r   r   	operationFTN)r*   rK   rE   r   r"   r   r   r   rJ   r+   r   r   r   rL   r   ADDr   UPSERTDELETEr   UPDATEr   )r5   r   rT   r   rQ   rR   rS   r6   r6   r7   r?     s0   
	"z*SqliteMetadataSegment._where_map_criterionrs   rZ   c                    s  g }|  D ]t\}}|dkr* fddttt |D }|tdd | q|dkrJ fddttt |D }|tdd | qttttt	tf f |}	j
 jjt|kt|	}
|j|
 qtd	d |S )
N$andc                       g | ]
} | qS r6   rn   r   wrZ   rs   rQ   r5   r6   r7   r         z>SqliteMetadataSegment._where_map_criterion.<locals>.<listcomp>c                 S      | |@ S rA   r6   xyr6   r6   r7   ry         z<SqliteMetadataSegment._where_map_criterion.<locals>.<lambda>$orc                    r   r6   r   r   r   r6   r7   r   	  r   c                 S      | |B S rA   r6   r   r6   r6   r7   ry     r   c                 S   r   rA   r6   r   r6   r6   r7   ry     r   )r   r   r   r   appendr'   r   r   r	   r    r*   rE   rF   rG   r1   rH   rf   r   _where_clauserp   )r5   rQ   rH   rs   rZ   clauser   r   criteriaexprsqr6   r   r7   rn     s*   z*SqliteMetadataSegment._where_doc_criterionrt   c                    sR  |  D ]\}}|dkr' fddttt |D }tdd |  S |dkrF fddttt |D }tdd |  S |d	krrtt|}d
| d
}	j 	j
jt|	}
j|
  S |dkrtt|}d
| d
}	j 	j
jt|	}
 j|
  S td| td)Nr   c              	         g | ]} | qS r6   ro   r   rZ   rt   rs   rQ   r5   r6   r7   r   '      
z>SqliteMetadataSegment._where_doc_criterion.<locals>.<listcomp>c                 S   r   rA   r6   r   r6   r6   r7   ry   -  r   z<SqliteMetadataSegment._where_doc_criterion.<locals>.<lambda>r   c              	      r   r6   r   r   r   r6   r7   r   /  r   c                 S   r   rA   r6   r   r6   r6   r7   ry   5  r   z	$contains%z$not_containszUnknown where_doc operator zEmpty where_doc)r   r   r   r   r'   r   r*   rE   rF   rG   r   rH   rg   liker   r1   rp   not_likerb   )r5   rQ   rH   rs   rt   rZ   r   r   r   search_termr   r6   r   r7   ro     sB   

zSqliteMetadataSegment.deletec                 C   sb  t d}t d}t d}| j | |j| j ||j|j	t
| j| jk}| j | |j| j ||j|j	t
| j| jk}| j | |j| j ||j|j	t
| j| jk}| j }|jt|  |jt|  |jt|  W d    d S 1 sw   Y  d S )NrV   r`   ra   )r"   r*   rE   rF   r   rH   r1   rp   rG   rI   r   rJ   r+   r   rK   rL   r   )r5   rP   t1t2q0rQ   r   rT   r6   r6   r7   r   R  sd   "r   )NNNNN)1__name__
__module____qualname__r   __annotations__r   r!   r   r   r   r   r8   r   r   ALLr   r<   rC   r   r=   rY   r[   r   r   r   r   r   rw   r   r$   r   rr   r   r   r{   r   r   r   r   r   r   r   r   r?   r"   r%   rn   ro   r   r6   r6   r6   r7   r(   +   s   
 

	



i

 
"
X
3

5r(   r@   r9   c                 C   sB   |   dkrt| ddS |   dkrt| ddS td|  )z Encode a SeqID into a byte array@      big      zUnsupported SeqID: )
bit_lengthrY   to_bytesrb   )r@   r6   r6   r7   r     s
   r   seq_id_bytesc                 C   sB   t | dkrt| dS t | dkrt| dS tdt |  )z Decode a byte array into a SeqIDr   r   r   zUnknown SeqID type with length )lenrY   
from_bytesrb   )r   r6   r6   r7   rO     s
   rO   r   tablec                 C   sF   t | ttttfrtttd| i|S tt	| 
 \}}t|||S )zLGiven a field name, an expression, and a table, construct a Pypika Criterion$eq)r   r   rY   r   r   r   r   r    nextrz   r   _value_criterion)r   r   operatorr   r6   r6   r7   r     s   r   r   opc                    s  t  tr
|jg}nt  tr|dv r|jg}nt  tr&|dv r&|jg}nt  tr4|dv r4|jg}nt  t	r|dv r }t
|dkrMtd| dt  d trj|dkr`|jt|n|jt|g}nt  d tr|dkr}|jt|n|jt|g}nrt  d tr|dkr|jt|n|jt|g}nUt  d tr|dkr|jt|n|jt|g}n8t  t	r|dv r|dkr|jt n|jt |dkr|jt n|jt g}n|j|jg}|dkr fdd	|D }nJ|d
kr fdd	|D }n;|dkr& fdd	|D }n,|dkr5 fdd	|D }n|dkrD fdd	|D }n|dkrR fdd	|D }|d
kr^tdd |S tdd |S )zmReturn a criterion to compare a value with the appropriate columns given its type
    and the operation type.)r   $ne)$inz$ninr   zEmpty list for z	 operatorr   r   c                    s   g | ]}|t  kqS r6   r   r   colr   r6   r7   r         z$_value_criterion.<locals>.<listcomp>r   c                    s   g | ]}|t  kqS r6   r   r   r   r6   r7   r     r   z$gtc                    s   g | ]}|t  kqS r6   r   r   r   r6   r7   r     r   z$gtec                    s   g | ]}|t  kqS r6   r   r   r   r6   r7   r     r   z$ltc                    s   g | ]}|t  k qS r6   r   r   r   r6   r7   r     r   z$ltec                    s   g | ]}|t  kqS r6   r   r   r   r6   r7   r     r   c                 S   r   rA   r6   r   r6   r6   r7   ry     r   z"_value_criterion.<locals>.<lambda>c                 S   r   rA   r6   r   r6   r6   r7   ry     r   )r   r   rg   r   rj   rY   rh   r   ri   rq   r   rb   rp   r   notinr'   )r   r   r   cols_v	col_exprsr6   r   r7   r     sr   




	






r   )Ctypingr   r   r   r   r   r   r   r	   r
   chromadb.segmentr   chromadb.ingestr   chromadb.configr   chromadb.typesr   r   chromadb.db.impl.sqliter   	overridesr   chromadb.db.baser   r   r    chromadb.telemetry.opentelemetryr   r   r   r   r   r   r   r   r   r   r   r    uuidr!   pypikar"   r#   pypika.queriesr$   pypika.functions	functionsrW   pypika.termsr%   	itertoolsr&   	functoolsr'   r   logging	getLoggerr   r   r(   bytesr   rO   r   r   r6   r6   r6   r7   <module>   s`   , ,
    d




