o
    3Iha)                     @   sn  d dl mZ d dlmZmZmZmZmZmZ d dl	Z	d dl
mZmZmZ d dlZd dlmZmZ d dlmZ d dlmZ d dlmZ d d	lmZmZmZ d d
lmZmZmZm Z m!Z! d dl"m#Z#m$Z$m%Z%m&Z&m'Z' d dl(m)Z)m*Z* d dl+Z+d dl,m-Z- d dl.m/Z/ e+0 dkrd dl1Z1n
e+0 dkrd dl2Z2ej3dej4dej5diZ6G dd deZ7dede dedefddZ8dS )    )Lock)SegmentImplementationSegmentManagerMetadataReaderSegmentTypeVectorReaderSN)SegmentLRUCache
BasicCacheSegmentCache)System	get_class)SysDB)override)PersistentLocalHnswSegment)OpenTelemetryClientOpenTelemetryGranularitytrace_method)
Collection	OperationSegmentSegmentScopeMetadata)DictTypeSequenceOptionalcast)UUIDuuid4)LRUCache)get_directory_sizeWindowsz;chromadb.segment.impl.metadata.sqlite.SqliteMetadataSegmentz8chromadb.segment.impl.vector.local_hnsw.LocalHnswSegmentzMchromadb.segment.impl.vector.local_persistent_hnsw.PersistentLocalHnswSegmentc                       s  e Zd ZU eed< eed< eed< eee	f ed< e
eef ed< ejZeed< eed< eed< d	ef fd
dZedejdeddfddZed, fddZed, fddZed, fddZedejededee fddZedejededee fddZdedefddZedejded e defd!dZ!ed"ejded#e"e# de#fd$d"Z$ed%ejeded&e%ddfd'd%Z&dede"e	 fd(d)Z'dede	fd*d+Z(  Z)S )-LocalSegmentManager_sysdb_system_opentelemetry_client
_instances#_vector_instances_file_handle_cache_vector_segment_type_lock_max_file_handlessystemc                    s  t  |  t _| _|t _t	t
 _i  _tjt i _|jjdkrG|jjdkrGt|jj fdd fddd jtj< nt  jtj< t  _ jjdrtj _t dkrmttjd  _ nt!j"j#$  _  j t%&  }t'|d	d d
 _(d S d S )NLRUr   c                    s
     |S N)callback_cache_evict)kvself c/home/air/sanwanet/gpt-api/venv/lib/python3.10/site-packages/chromadb/segment/impl/manager/local.py<lambda>N      
 z.LocalSegmentManager.__init__.<locals>.<lambda>c                    s
     | S r.   )_get_segment_disk_size)r0   r2   r4   r5   r6   O   r7   )capacitycallback	size_funcis_persistentr"   c                 S   s   |  S r.   )close_persistent_index)_r1   r4   r4   r5   r6   d   s    )r:   ))super__init__requirer   r$   r%   r   r&   logging	getLogger__name__loggerr'   r   METADATAr
   segment_cachesettingschroma_segment_cache_policychroma_memory_limit_bytesr	   VECTORr   r*   r   HNSW_LOCAL_PERSISTEDr)   platformr,   resource	getrlimitRLIMIT_NOFILEr+   ctypeswindllmsvcrt_getmaxstdior   get_file_handle_countr    r(   )r3   r,   segment_limit	__class__r2   r5   r@   >   s<   

zLocalSegmentManager.__init__z(LocalSegmentManager.callback_cache_evictsegmentreturnNc                 C   s<   |d }| j d|  | |}|  | j|d = d S )N
collectionzLRU cache evict collection id)rE   info	_instancestopr'   )r3   rY   collection_idinstancer4   r4   r5   r/   g   s
   
c                    &   | j  D ]}|  qt   d S r.   )r'   valuesstartr?   r3   ra   rW   r4   r5   rd   r      
zLocalSegmentManager.startc                    rb   r.   )r'   rc   r_   r?   re   rW   r4   r5   r_   x   rf   zLocalSegmentManager.stopc                    sD   | j  D ]
}|  |  qi | _ | jtj   t   d S r.   )	r'   rc   r_   reset_staterG   r   rK   resetr?   re   rW   r4   r5   rg   ~   s   
zLocalSegmentManager.reset_statez7LocalSegmentManager.prepare_segments_for_new_collectionr[   c                 C   s(   t | jtj|}t tjtj|}||gS r.   )_segmentr)   r   rK   r   SQLITErF   )r3   r[   vector_segmentmetadata_segmentr4   r4   r5   #prepare_segments_for_new_collection   s   

z#LocalSegmentManager.delete_segmentsr`   c                 C   s   | j j|d}|D ]T}|d | jv r=|d tjjkr%| |t}|  n|d tj	jkr7| |t
}|  | j|d = |d tju rM| jtj | |d tju r]| jtj | q	dd |D S )N)r[   r\   typescopec                 S   s   g | ]}|d  qS )r\   r4   ).0sr4   r4   r5   
<listcomp>   s    z7LocalSegmentManager.delete_segments.<locals>.<listcomp>)r$   get_segmentsr'   r   rL   valueget_segmentr   deleterj   r   r   rK   rG   poprF   )r3   r`   segmentsrY   ra   r4   r4   r5   delete_segments   s    
c                 C   sN   | j j|tjd}t|dkrdS ttj| j	j
dt|d d }|S )Nr[   ro   r   persist_directoryr\   )r$   rs   r   rK   lenr!   ospathjoinr%   rH   rA   str)r3   r`   rx   sizer4   r4   r5   r8      s   z*LocalSegmentManager._get_segment_disk_sizez&LocalSegmentManager._get_segment_sysdbro   c                    s@   | j j||d}tdd t D  tt fdd|}|S )Nrz   c                 S   s   g | ]}|j qS r4   )rt   )rp   r0   r4   r4   r5   rr      s    z:LocalSegmentManager._get_segment_sysdb.<locals>.<listcomp>c                    s   | d  v S Nrn   r4   )rq   known_typesr4   r5   r6      s    z8LocalSegmentManager._get_segment_sysdb.<locals>.<lambda>)r$   rs   setSEGMENT_TYPE_IMPLSkeysnextfilter)r3   r`   ro   rx   rY   r4   r   r5   _get_segment_sysdb   s   zLocalSegmentManager.get_segmentrn   c                 C   s   |t krtj}n|tkrtj}ntd| | j| |}|d u r2| ||}| j| 	|| | j
 | |}W d    n1 sEw   Y  tt|S )NzInvalid segment type: )r   r   rF   r   rK   
ValueErrorrG   getr   r   r*   r^   r   r   )r3   r`   rn   ro   rY   ra   r4   r4   r5   ru      s   
z'LocalSegmentManager.hint_use_collection	hint_typec                 C   sT   t tfD ]#}| ||}|tkr'| jjdr'tt|}|  | j	
|| qd S )Nr<   )r   r   ru   r%   rH   rA   r   r   open_persistent_indexr(   r   )r3   r`   r   rn   ra   r4   r4   r5   hint_use_collection   s   
c                 C   s   t t|d  }t|t}|S r   )r   r   r   r   )r3   rY   	classnameclsr4   r4   r5   _cls   s   
zLocalSegmentManager._clsc                 C   sH   |d | j vr| |}|| j|}|  || j |d < | j |d  S )Nr\   )r'   r   r%   rd   )r3   rY   r   ra   r4   r4   r5   r^      s   
zLocalSegmentManager._instance)rZ   N)*rD   
__module____qualname__r   __annotations__r   r   r   r   r   r    r   r   HNSW_LOCAL_MEMORYr)   r   intr@   r   r   OPERATION_AND_SEGMENTr   r/   r   rd   r_   rg   r   r   rm   ry   r8   r   r   r   r   ru   r   r   r   r^   __classcell__r4   r4   rW   r5   r#   2   st   
 )r#   rn   ro   r[   rZ   c                 C   s@   t t|  t}|j}d}|r||}tt | j||j|i dS )zRCreate a metadata dict, propagating metadata correctly for the given segment type.N)r\   rn   ro   r[   metadata
file_paths)	r   r   r   r   propagate_collection_metadatar   r   rt   r\   )rn   ro   r[   r   collection_metadatar   r4   r4   r5   ri      s   
ri   )9	threadingr   chromadb.segmentr   r   r   r   r   r   rB   )chromadb.segment.impl.manager.cache.cacher	   r
   r   r}   chromadb.configr   r   chromadb.db.systemr   	overridesr   2chromadb.segment.impl.vector.local_persistent_hnswr    chromadb.telemetry.opentelemetryr   r   r   chromadb.typesr   r   r   r   r   typingr   r   r   r   r   uuidr   r   rM   chromadb.utils.lru_cacher    chromadb.utils.directoryr!   r,   rN   rQ   rj   r   rL   r   r#   ri   r4   r4   r4   r5   <module>   s6     
 M