o #`^h@sddlZddlZddlmZddlZddlZddlZddl Zddl m Z m Z m Z mZddlmZmZmZmZmZddlmZmZmZddlmZddlmZddlmZdd lmZdd lm Z dd l!m"Z"dd l#m$Z$m%Z%dd l&m'Z'ddl(m)Z)m*Z*m+Z+m,Z,e-e.Z/da0ddZ1dddde ddfddZ2d/ddZ3GdddZ4Gddde%Z5Gddde$Z6GdddZ7Gdd d e7Z8Gd!d"d"eZ9Gd#d$d$Z:Gd%d&d&Z;Gd'd(d(ZGd-d.d.Z?dS)0N)BytesIO)AwsCredentialsAwsCredentialsProviderAwsSigningAlgorithmAwsSigningConfig)ClientBootstrapClientTlsContextDefaultHostResolverEventLoopGroupTlsContextOptions)S3ClientS3RequestTlsMode S3RequestType)UNSIGNED)urlsplit)Config)NoCredentialsError)MB)TransferNotDoneError)BaseTransferFutureBaseTransferMeta)TransferManager)CallArgsOSUtils get_callbacksis_s3express_bucketcCs>tdurtj|}z|Wn tyYdSw|atSN)CRT_S3_PROCESS_LOCKawscrts3CrossProcessLockacquire RuntimeError)namecrt_lockr%L/home/air/shanriGPT/back/venv/lib/python3.10/site-packages/s3transfer/crt.pyacquire_crt_s3_process_lock8s   r'Tc Cst|}t|}t||} d} |rtjntj} |dur2t} |r'| j|dnd| _t | } | } t |d}t | |||| | |ddS)a :type region: str :param region: The region used for signing :type crt_credentials_provider: Optional[awscrt.auth.AwsCredentialsProvider] :param crt_credentials_provider: CRT AWS credentials provider to use to sign requests. If not set, requests will not be signed. :type num_threads: Optional[int] :param num_threads: Number of worker threads generated. Default is the number of processors in the machine. :type target_throughput: Optional[int] :param target_throughput: Throughput target in bytes per second. By default, CRT will automatically attempt to choose a target throughput that matches the system's maximum network throughput. Currently, if CRT is unable to determine the maximum network throughput, a fallback target throughput of ``1_250_000_000`` bytes per second (which translates to 10 gigabits per second, or 1.16 gibibytes per second) is used. To set a specific target throughput, set a value for this parameter. :type part_size: Optional[int] :param part_size: Size, in Bytes, of parts that files will be downloaded or uploaded in. :type use_ssl: boolean :param use_ssl: Whether or not to use SSL. By default, SSL is used. Note that not all services support non-ssl connections. :type verify: Optional[boolean/string] :param verify: Whether or not to verify SSL certificates. By default SSL certificates are verified. You can provide the following values: * False - do not validate SSL certificates. SSL will still be used (unless use_ssl is False), but SSL certificates will not be verified. * path/to/cert/bundle.pem - A filename of the CA cert bundle to use. Specify this argument if you want to use a custom CA cert bundle instead of the default one on your system. N) ca_filepathF) provided_throughput_target_bytesT) bootstrapregioncredential_provider part_sizetls_modetls_connection_optionsthroughput_target_gbpsenable_s3express) r r rr ENABLEDDISABLEDr &override_default_trust_store_from_path verify_peerrnew_connection_options_get_crt_throughput_target_gbpsr )r,crt_credentials_provider num_threadstarget_throughputr.use_sslverifyevent_loop_group host_resolverr+r0r/tls_ctx_optionsclient_tls_option target_gbpsr%r%r&create_s3_crt_clientQs84 rCcCsH|durtj}td||durd}n|dd}td||S)Nz-Recommended CRT throughput target in gbps: %sg$@r(iʚ;z'Using CRT throughput target in gbps: %s)rr&get_recommended_throughput_target_gbpsloggerdebug)r*rBr%r%r&r8s   r8c@seZdZejZejZejZdZejZd$ddZ ddZ ddZ d%d d Z d%d d Z d%d dZd&ddZddZddZddZddZddZddZd&ddZd d!Zd"d#ZdS)'CRTTransferManagerTNcCsH|durt|_||_t||j|_|j|_g|_t d|_ d|_ dS)aFA transfer manager interface for Amazon S3 on CRT s3 client. :type crt_s3_client: awscrt.s3.S3Client :param crt_s3_client: The CRT s3 client, handling all the HTTP requests and functions under then hood :type crt_request_serializer: s3transfer.crt.BaseCRTRequestSerializer :param crt_request_serializer: Serializer, generates unsigned crt HTTP request. :type osutil: s3transfer.utils.OSUtils :param osutil: OSUtils object to use for os-related behavior when using with transfer manager. Nr) r_osutil_crt_s3_clientS3ClientArgsCreator_s3_args_creatortranslate_crt_exception_crt_exception_translator_future_coordinators threading Semaphore _semaphore _id_counter)self crt_s3_clientcrt_request_serializerosutilr%r%r&__init__s  zCRTTransferManager.__init__cCs|Srr%rTr%r%r& __enter__szCRTTransferManager.__enter__cGsd}|rd}||dS)NFT _shutdown)rTexc_type exc_valueargscancelr%r%r&__exit__szCRTTransferManager.__exit__cCsN|duri}|dur i}|||j||t|||||d}|d|S)Nbucketkeyfileobj extra_args subscribers get_object)_validate_all_known_argsALLOWED_DOWNLOAD_ARGS_validate_if_bucket_supportedr_submit_transfer)rTrcrdrerfrgcallargsr%r%r&downloads  zCRTTransferManager.downloadcCsX|duri}|dur i}|||j||||t|||||d}|d|S)Nrb put_object)riALLOWED_UPLOAD_ARGSrk&_validate_checksum_algorithm_supportedrrl)rTrercrdrfrgrmr%r%r&uploads   zCRTTransferManager.uploadcCsL|duri}|dur i}|||j||t||||d}|d|S)N)rcrdrfrg delete_object)riALLOWED_DELETE_ARGSrkrrl)rTrcrdrfrgrmr%r%r&delete s  zCRTTransferManager.deleteFcCs||dSrr[rTr`r%r%r&shutdownzCRTTransferManager.shutdowncCs@|jr|jD]\}}||}|rtd|dqdSdS)Nz'TransferManager methods do not support z+ resource. Use direct client calls instead.) VALIDATE_SUPPORTED_BUCKET_VALUES_UNSUPPORTED_BUCKET_PATTERNSitemsmatch ValueError)rTrcresourcepatternr|r%r%r&rks  z0CRTTransferManager._validate_if_bucket_supportedcCs0|D]}||vrtd|dd|qdS)NzInvalid extra_args key 'z', must be one of: z, )r}join)rTactualallowedkwargr%r%r&ri)sz+CRTTransferManager._validate_all_known_argscCsH|d}|dur dSttjjj}||vr"td|d|dS)NChecksumAlgorithmzChecksumAlgorithm: z* not supported. Supported algorithms are: )getlistrrS3ChecksumAlgorithm __members__upperr})rTrfchecksum_algorithmsupported_algorithmsr%r%r&rq1s  z9CRTTransferManager._validate_checksum_algorithm_supportedcCs |jD] }|s |qdSr)rOdoner`rT coordinatorr%r%r&_cancel_transfers<s z$CRTTransferManager._cancel_transferscC|jD]}|qdSr)rOresultrr%r%r&_finish_transfersA  z$CRTTransferManager._finish_transferscCrr)rO%wait_until_on_done_callbacks_completerr%r%r&_wait_transfers_doneErz'CRTTransferManager._wait_transfers_donecCsz|r|z1z|Wnty|Ynty"YnwW|dSW|dSW|dS|wr)rrKeyboardInterrupt Exceptionrrvr%r%r&r\Is      zCRTTransferManager._shutdowncK|jdSr)rRreleaserTkwargsr%r%r&_release_semaphoreVrxz%CRTTransferManager._release_semaphorec Cs|jg}t|j|jd}t|j||d}td i|}t|}||z$|j |j |d}||j |||||} |j jd i| } Wn&tyo} z|| d|j j |d|d} | | dWYd} ~ n d} ~ ww|| |j||jd7_|S) N) transfer_idexception_translator)metarqueuedTr)after_subscribers)errorr%)rCRTTransferCoordinatorrSrNCRTTransferMetaCRTTransferFutureAfterDoneHandlerappendrRr!rLget_crt_callbackget_make_request_argsrJ make_requestr set_exceptionset_s3_requestrO) rT request_type call_argson_done_after_callsr componentsfuture afterdone on_queued crt_callargscrt_s3_requesteon_doner%r%r&rlYsJ      z#CRTTransferManager._submit_transferrNNF)__name__ __module__ __qualname__rrjrprtryrzrXrZrarnrrrurwrkrirqrrrr\rrlr%r%r%r&rGs.        rGc@s>eZdZdZd ddZeddZeddZed d ZdS) rz*Holds metadata about the CRTTransferFutureNcCs||_||_i|_dSr) _transfer_id _call_args _user_context)rTrrr%r%r&rXs zCRTTransferMeta.__init__cC|jSr)rrYr%r%r&rzCRTTransferMeta.call_argscCrr)rrYr%r%r&rrzCRTTransferMeta.transfer_idcCrr)rrYr%r%r& user_contextrzCRTTransferMeta.user_contextr) rrr__doc__rXpropertyrrrr%r%r%r&rs   rc@sDeZdZdddZeddZddZddd Zd d Zd d Z dS)rNcCs ||_|dur t|_||_dS)aZThe future associated to a submitted transfer request via CRT S3 client :type meta: s3transfer.crt.CRTTransferMeta :param meta: The metadata associated to the transfer future. :type coordinator: s3transfer.crt.CRTTransferCoordinator :param coordinator: The coordinator associated to the transfer future. N)_metar _coordinator)rTrrr%r%r&rXs  zCRTTransferFuture.__init__cCrr)rrYr%r%r&rrzCRTTransferFuture.metacCs |jSr)rrrYr%r%r&r zCRTTransferFuture.donecC|j|dSr)rrrTtimeoutr%r%r&rzCRTTransferFuture.resultcCrr)rr`rYr%r%r&r`rxzCRTTransferFuture.cancelcCs$|std|jj|dddS)z!Sets the exception on the future.z?set_exception can only be called once the transfer is complete.T)overrideN)rrrrrT exceptionr%r%r&rs zCRTTransferFuture.set_exceptionrr) rrrrXrrrrr`rr%r%r%r&rs    rc@eZdZddZddZdS)BaseCRTRequestSerializercCtd)alSerialize CRT HTTP requests. :type transfer_type: string :param transfer_type: the type of transfer made, e.g 'put_object', 'get_object', 'delete_object' :type future: s3transfer.crt.CRTTransferFuture :rtype: awscrt.http.HttpRequest :returns: An unsigned HTTP request to be used for the CRT S3 client zserialize_http_request()NotImplementedError)rT transfer_typerr%r%r&serialize_http_requests z/BaseCRTRequestSerializer.serialize_http_requestcCr)Nztranslate_crt_exception()rrr%r%r&rMsz0BaseCRTRequestSerializer.translate_crt_exceptionN)rrrrrMr%r%r%r&rs rc@sfeZdZdddZddZddZdd Zd d Zd d ZddZ ddZ ddZ ddZ ddZ dS)BotocoreCRTRequestSerializerNcCsp||_|dur i}||||jdi||_|jjjd|j|jjjd|j|jjjd|j dS)aSerialize CRT HTTP request using botocore logic It also takes into account configuration from both the session and any keyword arguments that could be passed to `Session.create_client()` when serializing the request. :type session: botocore.session.Session :type client_kwargs: Optional[Dict[str, str]]) :param client_kwargs: The kwargs for the botocore s3 client initialization. Nzrequest-created.s3.*zafter-call.s3.*zbefore-send.s3.*r%) _session_resolve_client_config create_client_clientreventsregister_capture_http_request+_change_response_to_serialized_http_request_make_fake_http_response)rTsession client_kwargsr%r%r&rXs     z%BotocoreCRTRequestSerializer.__init__cCsPd}|r |}d|vr|d}ttd}|r||}||d<d|d<dS)Nconfig)signature_versionr service_name)get_default_client_configrrmerge)rTrruser_provided_config client_configr%r%r&rs   z3BotocoreCRTRequestSerializer._resolve_client_configc Cst|j}|j}|jr|d|j}g}|jD]\}}t|tr+|||fq||t|dfqt j |}t j j |j |||jd}|S)N?zutf-8)methodpathheaders body_stream)rurlrqueryrr{ isinstancestrrrhttp HttpHeaders HttpRequestrbody) rT aws_request url_partscrt_path headers_listr#value crt_headers crt_requestr%r%r&_crt_request_from_aws_requests"   z:BotocoreCRTRequestSerializer._crt_request_from_aws_requestcCs||}|jddurt|j}|jd|j|jddur(|jd|jddur<|jdur<|j dd|jddurJ|jd|S)Nhostz Content-MD5zContent-Length0zTransfer-Encoding) rrrrrsetnetlocremoveradd)rTbotocore_http_requestrrr%r%r&_convert_to_crt_http_requests     z9BotocoreCRTRequestSerializer._convert_to_crt_http_requestcKs||jd<dS)N http_request)contextrTrequestrr%r%r&r.rxz2BotocoreCRTRequestSerializer._capture_http_requestcKs|d}||d<dS)Nr HTTPRequest)prepare)rTrparsedrr r%r%r&r1szHBotocoreCRTRequestSerializer._change_response_to_serialized_http_requestcKstjdditdS)N)botocore awsrequest AWSResponseFakeRawResponser r%r%r&r7s z5BotocoreCRTRequestSerializer._make_fake_http_responsecCs&t|j|d|j|jd|jdS)N)BucketKeyr r%)getattrrrcrdrf)rT client_methodrr%r%r&_get_botocore_http_request?s z7BotocoreCRTRequestSerializer._get_botocore_http_requestcCs|||jj}||}|Sr)rrrr)rTrrrrr%r%r&rDs  z3BotocoreCRTRequestSerializer.serialize_http_requestcCst|tjjr ||SdSr)rrrS3ResponseError _translate_crt_s3_response_errorrr%r%r&rMKs z4BotocoreCRTRequestSerializer.translate_crt_exceptionc Cs|j}|dkr dSdd|jD}|j}|dur$|jjj}||j}nd}tj |||j d}|jj j ||d}|did} |jj| } | ||dS) Ni-cSsi|]\}}||qSr%r%).0kvr%r%r& XszQBotocoreCRTRequestSerializer._translate_crt_s3_response_error..)r status_coder)shapeErrorCode)operation_name)rrr#rr service_modeloperation_model output_shaperr HeadersDictr_response_parserparser exceptions from_code) rTs3_response_errorrrr#r$r  response_dictparsed_response error_code error_classr%r%r&rQs&   z=BotocoreCRTRequestSerializer._translate_crt_s3_response_errorr)rrrrXrrrrrrrrrMrr%r%r%r&rs   rc@seZdZdddZdS)rNccs ||}|s dS|Vqr)read)rTamtdecode_contentchunkr%r%r&streamos zFakeRawResponse.stream)r1N)rrrr6r%r%r%r&rnsrc@s,eZdZddZddZddZddZd S) BotocoreCRTCredentialsWrappercC ||_dSr)_resolved_credentials)rTresolved_botocore_credentialsr%r%r&rXxrz&BotocoreCRTCredentialsWrapper.__init__cCs|}t|j|j|jSr)_get_credentialsget_frozen_credentialsr access_key secret_keytoken)rT credentialsr%r%r&__call__{s  z&BotocoreCRTCredentialsWrapper.__call__cCs t|Sr)r new_delegaterYr%r%r&to_crt_credentials_providerrz9BotocoreCRTCredentialsWrapper.to_crt_credentials_providercCs|jdurt|jSr)r9rrYr%r%r&r;s z.BotocoreCRTCredentialsWrapper._get_credentialsN)rrrrXrArCr;r%r%r%r&r7ws  r7c@sneZdZdZ dddZeddZddZdd d Zdd d Z ddZ dddZ ddZ ddZ ddZdS)rz-A helper class for managing CRTTransferFutureNcCs6||_||_||_t|_d|_d|_t|_ dSr) r_exception_translator _s3_requestrPLock_lock _exception _crt_futureEvent _done_event)rTr s3_requestrr%r%r&rXs zCRTTransferCoordinator.__init__cCrr)rErYr%r%r&rLrz!CRTTransferCoordinator.s3_requestcCrr)rKrrYr%r%r&set_done_callbacks_completerxz2CRTTransferCoordinator.set_done_callbacks_completecCrr)rKwaitrr%r%r&rrz.invoke_all_callbacksr%)rTrrbrarrcr%r`r&rsz$S3ClientArgsCreator.get_crt_callbackc Csd}t|jtr|j}|j|}||jd<n|j|jd<|jdd} tj j tj j | tj j j d} d|jd<|j||||||d} || d <| | d <| S) N ContentLengthBodyrCRC32) algorithmlocationzoverride-to-be-removed ContentMD5rX send_filepathchecksum_config)rrerrU get_file_sizerfpoprrrS3ChecksumConfigrS3ChecksumLocationTRAILERrZ) rTrrrrrYrrjdata_lenrrkmake_request_argsr%r%r&!_get_make_request_args_put_objects6      z5S3ClientArgsCreator._get_make_request_args_put_objectc Csd}d}tjjdd} t|jtr&|j} |j| }|t || ||jnt |j}|j ||||||d} || d<|| d<| | d<| S)NT)validate_responserX recv_filepathon_bodyrk) rrrnrrerrUget_temp_filenamerRenameTempFileHandlerOnBodyFileObjWriterrZ) rTrrrrrYrrurvrkfinal_filepathrrr%r%r&!_get_make_request_args_get_object,s0     z5S3ClientArgsCreator._get_make_request_args_get_objectcCs|j||tt|tj||d||||dd}|dtjkr3ddd|dD|d <t |j r@t t j d |d <|S) Nrr\)r typer on_progressr|css|]}|VqdSr)title)rxr%r%r& is zES3ClientArgsCreator._default_get_make_request_args.._r#)rgsigning_config)rTrrrrDEFAULTrrsplitrrcrr V4_S3EXPRESS)rTrrrrrYrrrr%r%r&rZPs(      z2S3ClientArgsCreator._default_get_make_request_argsr) rrrrXrrrsr{rZr%r%r%r&rKs * $rKc@r)rxcCs||_||_||_||_dSr)r_final_filename_temp_filenamerI)rTrfinal_filename temp_filenamerWr%r%r&rXus zRenameTempFileHandler.__init__c Ksx|d}|r|j|jdSz |j|j|jWdSty;}z|j|j|j|WYd}~dSd}~ww)Nr)rI remove_filer rename_filerrrr)rTrrrr%r%r&rA{s zRenameTempFileHandler.__call__NrrrrXrAr%r%r%r&rxts rxc@r)rcCr8r)rrr%r%r&rXrzAfterDoneHandler.__init__cKrr)rrMrr%r%r&rArxzAfterDoneHandler.__call__Nrr%r%r%r&r rc@r)rycCr8r)_fileobj)rTrer%r%r&rXrzOnBodyFileObjWriter.__init__cKrr)rwrite)rTr5rr%r%r&rArzOnBodyFileObjWriter.__call__Nrr%r%r%r&ryrryr)@loggingrPior awscrt.httpr awscrt.s3botocore.awsrequestrbotocore.session awscrt.authrrrr awscrt.iorrr r r r r rrbotocore.compatrbotocore.configrbotocore.exceptionsrs3transfer.constantsrs3transfer.exceptionsrs3transfer.futuresrrs3transfer.managerrs3transfer.utilsrrrr getLoggerrrErr'rCr8rGrrrrrr7rrKrxrryr%r%r%r&sZ           UL& I"