o
    #`^h)w                     @   s   d dl Z d dlmZ d dlmZmZ d dlmZ d dlm	Z	m
Z
mZmZ d dlmZmZmZmZ G dd dZG d	d
 d
ZG dd dZG dd deZG dd deZG dd deZG dd deZG dd deZG dd deZdS )    N)BytesIOreadableseekable)IN_MEMORY_UPLOAD_TAG)CompleteMultipartUploadTaskCreateMultipartUploadTaskSubmissionTaskTask)ChunksizeAdjusterDeferredOpenFileget_callbacksget_filtered_dictc                   @   s.   e Zd ZdddZdd Zdd Zdd	 Zd
S )AggregatedProgressCallback   c                 C   s   || _ || _d| _dS )a  Aggregates progress updates for every provided progress callback

        :type callbacks: A list of functions that accepts bytes_transferred
            as a single argument
        :param callbacks: The callbacks to invoke when threshold is reached

        :type threshold: int
        :param threshold: The progress threshold in which to take the
            aggregated progress and invoke the progress callback with that
            aggregated progress total
        r   N)
_callbacks
_threshold_bytes_seen)self	callbacks	threshold r   O/home/air/shanriGPT/back/venv/lib/python3.10/site-packages/s3transfer/upload.py__init__!   s   
z#AggregatedProgressCallback.__init__c                 C   s*   |  j |7  _ | j | jkr|   d S d S N)r   r   _trigger_callbacks)r   bytes_transferredr   r   r   __call__1   s   z#AggregatedProgressCallback.__call__c                 C   s   | j dkr|   dS dS )z@Flushes out any progress that has not been sent to its callbacksr   N)r   r   r   r   r   r   flush6   s   
z AggregatedProgressCallback.flushc                 C   s"   | j D ]}|| jd qd| _d S )N)r   r   )r   r   )r   callbackr   r   r   r   ;   s   

z-AggregatedProgressCallback._trigger_callbacksN)r   )__name__
__module____qualname__r   r   r   r   r   r   r   r   r       s
    
r   c                   @   sL   e Zd ZdZdd ZdddZddd	Zd
d Zdd Zdd Z	dd Z
dS )InterruptReadera  Wrapper that can interrupt reading using an error

    It uses a transfer coordinator to propagate an error if it notices
    that a read is being made while the file is being read from.

    :type fileobj: file-like obj
    :param fileobj: The file-like object to read from

    :type transfer_coordinator: s3transfer.futures.TransferCoordinator
    :param transfer_coordinator: The transfer coordinator to use if the
        reader needs to be interrupted.
    c                 C   s   || _ || _d S r   )_fileobj_transfer_coordinator)r   fileobjtransfer_coordinatorr   r   r   r   O   s   
zInterruptReader.__init__Nc                 C   s   | j jr| j j| j|S r   )r&   	exceptionr%   read)r   amountr   r   r   r*   S   s   zInterruptReader.readr   c                 C   s   | j || d S r   )r%   seek)r   wherewhencer   r   r   r,   ]   s   zInterruptReader.seekc                 C   s
   | j  S r   )r%   tellr   r   r   r   r/   `   s   
zInterruptReader.tellc                 C   s   | j   d S r   )r%   closer   r   r   r   r0   c      zInterruptReader.closec                 C   s   | S r   r   r   r   r   r   	__enter__f      zInterruptReader.__enter__c                 O   s   |    d S r   )r0   )r   argskwargsr   r   r   __exit__i      zInterruptReader.__exit__r   )r   )r!   r"   r#   __doc__r   r*   r,   r/   r0   r2   r6   r   r   r   r   r$   A   s    


r$   c                   @   sf   e Zd ZdZdddZedd Zdd Zd	d
 Zdd Z	dd Z
dd Zdd Zdd Zdd ZdS )UploadInputManageraJ  Base manager class for handling various types of files for uploads

    This class is typically used for the UploadSubmissionTask class to help
    determine the following:

        * How to determine the size of the file
        * How to determine if a multipart upload is required
        * How to retrieve the body for a PutObject
        * How to retrieve the bodies for a set of UploadParts

    The answers/implementations differ for the various types of file inputs
    that may be accepted. All implementations must subclass and override
    public methods from this class.
    Nc                 C   s   || _ || _|| _d S r   )_osutilr&   _bandwidth_limiterr   osutilr(   bandwidth_limiterr   r   r   r   }   s   
zUploadInputManager.__init__c                 C      t d)a  Determines if the source for the upload is compatible with manager

        :param upload_source: The source for which the upload will pull data
            from.

        :returns: True if the manager can handle the type of source specified
            otherwise returns False.
        zmust implement _is_compatible()NotImplementedErrorclsupload_sourcer   r   r   is_compatible   s   
z UploadInputManager.is_compatiblec                 C   r?   )a  Whether the body it provides are stored in-memory

        :type operation_name: str
        :param operation_name: The name of the client operation that the body
            is being used for. Valid operation_names are ``put_object`` and
            ``upload_part``.

        :rtype: boolean
        :returns: True if the body returned by the manager will be stored in
            memory. False if the manager will not directly store the body in
            memory.
        z%must implement store_body_in_memory()r@   r   operation_namer   r   r   stores_body_in_memory      z(UploadInputManager.stores_body_in_memoryc                 C   r?   )zProvides the transfer size of an upload

        :type transfer_future: s3transfer.futures.TransferFuture
        :param transfer_future: The future associated with upload request
        z&must implement provide_transfer_size()r@   r   transfer_futurer   r   r   provide_transfer_size   s   z(UploadInputManager.provide_transfer_sizec                 C   r?   )a  Determines where a multipart upload is required

        :type transfer_future: s3transfer.futures.TransferFuture
        :param transfer_future: The future associated with upload request

        :type config: s3transfer.manager.TransferConfig
        :param config: The config associated to the transfer manager

        :rtype: boolean
        :returns: True, if the upload should be multipart based on
            configuration and size. False, otherwise.
        z*must implement requires_multipart_upload()r@   r   rK   configr   r   r   requires_multipart_upload   rI   z,UploadInputManager.requires_multipart_uploadc                 C   r?   )a  Returns the body to use for PutObject

        :type transfer_future: s3transfer.futures.TransferFuture
        :param transfer_future: The future associated with upload request

        :type config: s3transfer.manager.TransferConfig
        :param config: The config associated to the transfer manager

        :rtype: s3transfer.utils.ReadFileChunk
        :returns: A ReadFileChunk including all progress callbacks
            associated with the transfer future.
        z$must implement get_put_object_body()r@   rJ   r   r   r   get_put_object_body   rI   z&UploadInputManager.get_put_object_bodyc                 C   r?   )a  Yields the part number and body to use for each UploadPart

        :type transfer_future: s3transfer.futures.TransferFuture
        :param transfer_future: The future associated with upload request

        :type chunksize: int
        :param chunksize: The chunksize to use for this upload.

        :rtype: int, s3transfer.utils.ReadFileChunk
        :returns: Yields the part number and the ReadFileChunk including all
            progress callbacks associated with the transfer future for that
            specific yielded part.
        z)must implement yield_upload_part_bodies()r@   )r   rK   	chunksizer   r   r   yield_upload_part_bodies   s   z+UploadInputManager.yield_upload_part_bodiesc                 C   s*   t || j}| jr| jj|| jdd}|S )NF)enabled)r$   r&   r;   get_bandwith_limited_stream)r   r'   r   r   r   _wrap_fileobj   s   z UploadInputManager._wrap_fileobjc                 C   s   t |d}|rt|gS g S )Nprogress)r   r   )r   rK   r   r   r   r   _get_progress_callbacks   s   

z*UploadInputManager._get_progress_callbacksc                 C   s   dd |D S )Nc                 S   s   g | ]}|j qS r   )r   ).0r    r   r   r   
<listcomp>   s    z;UploadInputManager._get_close_callbacks.<locals>.<listcomp>r   )r   aggregated_progress_callbacksr   r   r   _get_close_callbacks   r1   z'UploadInputManager._get_close_callbacksr   )r!   r"   r#   r8   r   classmethodrE   rH   rL   rO   rP   rR   rU   rW   r[   r   r   r   r   r9   m   s    

	r9   c                   @   sd   e Zd ZdZedd Zdd Zdd Zdd	 Zd
d Z	dd Z
dd Zdd Zdd Zdd ZdS )UploadFilenameInputManagerzUpload utility for filenamesc                 C   s
   t |tS r   )
isinstancestrrB   r   r   r   rE      s   
z(UploadFilenameInputManager.is_compatiblec                 C      dS )NFr   rF   r   r   r   rH      r3   z0UploadFilenameInputManager.stores_body_in_memoryc                 C   s   |j | j|j jj d S r   )metarL   r:   get_file_size	call_argsr'   rJ   r   r   r   rL      s   z0UploadFilenameInputManager.provide_transfer_sizec                 C   s   |j j|jkS r   )ra   sizemultipart_thresholdrM   r   r   r   rO      r1   z4UploadFilenameInputManager.requires_multipart_uploadc                 C   sJ   |  |\}}| |}| |}| |}|jj}| jj|||||dS )Nr'   
chunk_sizefull_file_sizer   close_callbacks)&_get_put_object_fileobj_with_full_sizerU   rW   r[   ra   rd   r:   #open_file_chunk_reader_from_fileobj)r   rK   r'   	full_sizer   ri   rd   r   r   r   rP      s   


z.UploadFilenameInputManager.get_put_object_bodyc                 c   s    |j j}| ||}td|d D ]5}| |}| |}||d  }| j|j jj|||d\}	}
| 	|	}	| j
j|	||
||d}||fV  qd S )N   )
start_byte	part_sizerh   rf   )ra   rd   _get_num_partsrangerW   r[   '_get_upload_part_fileobj_with_full_sizerc   r'   rU   r:   rk   )r   rK   rQ   rh   	num_partspart_numberr   ri   rn   r'   rl   read_file_chunkr   r   r   rR     s.   




z3UploadFilenameInputManager.yield_upload_part_bodiesc                 C   s   t ||| jjd}|S )N)open_function)r   r:   open)r   r'   rn   r   r   r   _get_deferred_open_file1  s   
z2UploadFilenameInputManager._get_deferred_open_filec                 C   s"   |j jj}|j j}| |d|fS )Nr   )ra   rc   r'   rd   rx   r   rK   r'   rd   r   r   r   rj   7  s   
zAUploadFilenameInputManager._get_put_object_fileobj_with_full_sizec                 K   s    |d }|d }|  |||fS )Nrn   rh   )rx   )r   r'   r5   rn   rl   r   r   r   rr   <  s   zBUploadFilenameInputManager._get_upload_part_fileobj_with_full_sizec                 C   s   t t|jjt| S r   )intmathceilra   rd   float)r   rK   ro   r   r   r   rp   A  s   z)UploadFilenameInputManager._get_num_partsN)r!   r"   r#   r8   r\   rE   rH   rL   rO   rP   rR   rx   rj   rr   rp   r   r   r   r   r]      s    
r]   c                   @   s<   e Zd ZdZedd Zdd Zdd Zdd	 Zd
d Z	dS )UploadSeekableInputManagerz&Upload utility for an open file objectc                 C   s   t |ot|S r   r   rB   r   r   r   rE   H  s   z(UploadSeekableInputManager.is_compatiblec                 C   s   |dkrdS dS )N
put_objectFTr   rF   r   r   r   rH   L  s   z0UploadSeekableInputManager.stores_body_in_memoryc                 C   sD   |j jj}| }|dd | }|| |j ||  d S )Nr      )ra   rc   r'   r/   r,   rL   )r   rK   r'   start_positionend_positionr   r   r   rL   R  s   

z0UploadSeekableInputManager.provide_transfer_sizec                 K   s   | |d }t|t|fS )Nro   )r*   r   len)r   r'   r5   datar   r   r   rr   _  s   zBUploadSeekableInputManager._get_upload_part_fileobj_with_full_sizec                 C   s"   |j jj}| |j j }||fS r   )ra   rc   r'   r/   rd   ry   r   r   r   rj   n  s   
zAUploadSeekableInputManager._get_put_object_fileobj_with_full_sizeN)
r!   r"   r#   r8   r\   rE   rH   rL   rr   rj   r   r   r   r   r~   E  s    
r~   c                       sh   e Zd ZdZd fdd	Zedd Zdd Zd	d
 Zdd Z	dd Z
dd ZdddZdd Z  ZS )UploadNonSeekableInputManagerz7Upload utility for a file-like object that cannot seek.Nc                    s   t  ||| d| _d S )N    )superr   _initial_datar<   	__class__r   r   r   y  s   
z&UploadNonSeekableInputManager.__init__c                 C   s   t |S r   )r   rB   r   r   r   rE   }  s   z+UploadNonSeekableInputManager.is_compatiblec                 C   r`   )NTr   rF   r   r   r   rH     r3   z3UploadNonSeekableInputManager.stores_body_in_memoryc                 C   s   d S r   r   rJ   r   r   r   rL     s   z3UploadNonSeekableInputManager.provide_transfer_sizec                 C   sP   |j jd ur|j j|jkS |j jj}|j}| ||d| _t| j|k r&dS dS )NFT)ra   rd   re   rc   r'   _readr   r   )r   rK   rN   r'   r   r   r   r   rO     s   
z7UploadNonSeekableInputManager.requires_multipart_uploadc                 C   s@   |  |}| |}|jjj}| | j|  ||}d | _|S r   )rW   r[   ra   rc   r'   
_wrap_datar   r*   )r   rK   r   ri   r'   bodyr   r   r   rP     s   


z1UploadNonSeekableInputManager.get_put_object_bodyc           	      c   s`    |j jj}d}	 | |}| |}|d7 }| ||}|s!d S | |||}d }||fV  q	)Nr   Trm   )ra   rc   r'   rW   r[   r   r   )	r   rK   rQ   file_objectrt   r   ri   part_contentpart_objectr   r   r   rR     s    



z6UploadNonSeekableInputManager.yield_upload_part_bodiesTc                 C   sx   t | jdkr||S |t | jkr&| jd| }|r$| j|d | _|S |t | j }| j|| }|r:d| _|S )a=  
        Reads a specific amount of data from a stream and returns it. If there
        is any data in initial_data, that will be popped out first.

        :type fileobj: A file-like object that implements read
        :param fileobj: The stream to read from.

        :type amount: int
        :param amount: The number of bytes to read from the stream.

        :type truncate: bool
        :param truncate: Whether or not to truncate initial_data after
            reading from it.

        :return: Generator which generates part bodies from the initial data.
        r   Nr   )r   r   r*   )r   r'   r+   truncater   amount_to_readr   r   r   r     s   
z#UploadNonSeekableInputManager._readc                 C   s,   |  t|}| jj|t|t|||dS )a  
        Wraps data with the interrupt reader and the file chunk reader.

        :type data: bytes
        :param data: The data to wrap.

        :type callbacks: list
        :param callbacks: The callbacks associated with the transfer future.

        :type close_callbacks: list
        :param close_callbacks: The callbacks to be called when closing the
            wrapper for the data.

        :return: Fully wrapped data.
        rf   )rU   r   r:   rk   r   )r   r   r   ri   r'   r   r   r   r     s   z(UploadNonSeekableInputManager._wrap_datar   )T)r!   r"   r#   r8   r   r\   rE   rH   rL   rO   rP   rR   r   r   __classcell__r   r   r   r   r   v  s    

*r   c                   @   s\   e Zd ZdZg dZg dZdd Z	dddZd	d
 Zdd Z	dd Z
dd Zdd ZdS )UploadSubmissionTaskz.Task for submitting tasks to execute an upload)ChecksumAlgorithmSSECustomerKeySSECustomerAlgorithmSSECustomerKeyMD5RequestPayerExpectedBucketOwner)r   r   r   r   r   c                 C   sD   t ttg}|jjj}|D ]}||r|  S qtd|t	|)ao  Retrieves a class for managing input for an upload based on file type

        :type transfer_future: s3transfer.futures.TransferFuture
        :param transfer_future: The transfer future for the request

        :rtype: class of UploadInputManager
        :returns: The appropriate class to use for managing a specific type of
            input for uploads.
        z&Input {} of type: {} is not supported.)
r]   r~   r   ra   rc   r'   rE   RuntimeErrorformattype)r   rK   upload_manager_resolver_chainr'   upload_manager_clsr   r   r   _get_upload_input_manager_cls  s   

z2UploadSubmissionTask._get_upload_input_manager_clsNc                 C   sf   |  ||| j|}|jjdu r|| |||s'| |||||| dS | |||||| dS )a  
        :param client: The client associated with the transfer manager

        :type config: s3transfer.manager.TransferConfig
        :param config: The transfer config associated with the transfer
            manager

        :type osutil: s3transfer.utils.OSUtil
        :param osutil: The os utility associated to the transfer manager

        :type request_executor: s3transfer.futures.BoundedExecutor
        :param request_executor: The request executor associated with the
            transfer manager

        :type transfer_future: s3transfer.futures.TransferFuture
        :param transfer_future: The transfer future associated with the
            transfer request that tasks are being submitted for
        N)r   r&   ra   rd   rL   rO   _submit_upload_request_submit_multipart_request)r   clientrN   r=   request_executorrK   r>   upload_input_managerr   r   r   _submit.  s4   
	zUploadSubmissionTask._submitc           	   
   C   sN   |j j}| |d}| jj|t| j||||j|j|j	ddd|d d S )Nr   )r   r'   bucketkey
extra_argsT)r(   main_kwargsis_finaltag)
ra   rc   _get_upload_task_tagr&   submitPutObjectTaskrP   r   r   r   )	r   r   rN   r=   r   rK   r   rc   put_object_tagr   r   r   r   g  s(   		
z+UploadSubmissionTask._submit_upload_requestc                 C   s   |j j}| j|t| j||j|j|jdd}g }	| |j}
| 	|d}|j j
}t }||j|}|||}|D ]!\}}|	| jj|t| j|||j|j||
dd|id|d q<| |j}| j|t| j||j|j|d||	dd	d
 d S )N)r   r   r   r   )r(   r   upload_part)r   r'   r   r   rt   r   	upload_id)r(   r   pending_main_kwargsr   )r   partsT)r(   r   r   r   )ra   rc   r&   r   r   r   r   r   _extra_upload_part_argsr   rd   r   adjust_chunksizemultipart_chunksizerR   appendUploadPartTask_extra_complete_multipart_argsr   )r   r   rN   r=   r   rK   r   rc   create_multipart_futurepart_futuresextra_part_argsupload_part_tagrd   adjusterrQ   part_iteratorrt   r'   complete_multipart_extra_argsr   r   r   r     sx   		z.UploadSubmissionTask._submit_multipart_requestc                 C      t || jS r   )r   UPLOAD_PART_ARGSr   r   r   r   r   r     s   z,UploadSubmissionTask._extra_upload_part_argsc                 C   r   r   )r   COMPLETE_MULTIPART_ARGSr   r   r   r   r     r7   z3UploadSubmissionTask._extra_complete_multipart_argsc                 C   s   d }| |r	t}|S r   )rH   r   )r   r   rG   r   r   r   r   r     s   
z)UploadSubmissionTask._get_upload_task_tagr   )r!   r"   r#   r8   r   r   r   r   r   r   r   r   r   r   r   r   r   r      s    	!
9#Vr   c                   @      e Zd ZdZdd ZdS )r   z Task to do a nonmultipart uploadc                 C   sB   |}|j d|||d| W d   dS 1 sw   Y  dS )aP  
        :param client: The client to use when calling PutObject
        :param fileobj: The file to upload.
        :param bucket: The name of the bucket to upload to
        :param key: The name of the key to upload to
        :param extra_args: A dictionary of any extra arguments that may be
            used in the upload.
        )BucketKeyBodyNr   )r   )r   r   r'   r   r   r   r   r   r   r   _main  s   	"zPutObjectTask._mainNr!   r"   r#   r8   r   r   r   r   r   r         r   c                   @   r   )r   z+Task to upload a part in a multipart uploadc              	   C   s   |}|j d|||||d|}	W d   n1 sw   Y  |	d }
|
|d}d|v rB|d  }d| }||	v rB|	| ||< |S )a  
        :param client: The client to use when calling PutObject
        :param fileobj: The file to upload.
        :param bucket: The name of the bucket to upload to
        :param key: The name of the key to upload to
        :param upload_id: The id of the upload
        :param part_number: The number representing the part of the multipart
            upload
        :param extra_args: A dictionary of any extra arguments that may be
            used in the upload.

        :rtype: dict
        :returns: A dictionary representing a part::

            {'Etag': etag_value, 'PartNumber': part_number}

            This value can be appended to a list to be used to complete
            the multipart upload.
        )r   r   UploadId
PartNumberr   NETag)r   r   r   Checksumr   )r   upper)r   r   r'   r   r   r   rt   r   r   responseetagpart_metadataalgorithm_namechecksum_memberr   r   r   r     s&   	

zUploadPartTask._mainNr   r   r   r   r   r     r   r   )r{   ior   s3transfer.compatr   r   s3transfer.futuresr   s3transfer.tasksr   r   r	   r
   s3transfer.utilsr   r   r   r   r   r$   r9   r]   r~   r   r   r   r   r   r   r   r   <module>   s"   !,{]1  p