U .bV@sddlZddlZddlmZddlZddlZddlZddl m Z m Z ddl m Z mZmZmZmZddlmZmZmZddlmZddlmZddlmZdd lmZdd lmZmZdd l m!Z!dd l"m#Z#m$Z$dd l%m&Z&m'Z'm(Z(e)e*Z+GdddZ,dddeddeddfddZ-GdddZ.Gddde$Z/Gddde#Z0GdddZ1Gddde1Z2Gdd d eZ3Gd!d"d"Z4Gd#d$d$Z5Gd%d&d&Z6Gd'd(d(Z7dS))N)BytesIO)AwsCredentialsAwsCredentialsProvider)ClientBootstrapClientTlsContextDefaultHostResolverEventLoopGroupTlsContextOptions)S3ClientS3RequestTlsMode S3RequestType)UNSIGNED)urlsplit)Config)NoCredentialsError)GBMB)TransferNotDoneError)BaseTransferFutureBaseTransferMeta)CallArgsOSUtils get_callbacksc@s$eZdZddZddZddZdS)CRTCredentialProviderAdaptercCs||_d|_t|_dSN)_botocore_credential_provider_loaded_credentials threadingLock_lock)selfbotocore_credential_providerr"2/tmp/pip-unpacked-wheel-hq9fdgne/s3transfer/crt.py__init__+sz%CRTCredentialProviderAdapter.__init__cCs|}t|j|j|jSr)_get_credentialsZget_frozen_credentialsrZ access_keyZ secret_keytoken)r credentialsr"r"r#__call__0s  z%CRTCredentialProviderAdapter.__call__c CsL|j<|jdkr0|j}|dkr*t||_|jW5QRSQRXdSr)rrrZload_credentialsr)r Z loaded_credsr"r"r#r%6s z-CRTCredentialProviderAdapter._get_credentialsN)__name__ __module__ __qualname__r$r(r%r"r"r"r#r*srTc Cst|}t|}t||} d} d} |r,tjntj} |dk rht} |rR| j|dnd| _t | }| } |r~t |}t |} |dt}t| || || | |dS)a :type region: str :param region: The region used for signing :type botocore_credential_provider: Optional[botocore.credentials.CredentialResolver] :param botocore_credential_provider: Provide credentials for CRT to sign the request if not set, the request will not be signed :type num_threads: Optional[int] :param num_threads: Number of worker threads generated. Default is the number of processors in the machine. :type target_throughput: Optional[int] :param target_throughput: Throughput target in Bytes. Default is 0.625 GB/s (which translates to 5 Gb/s). :type part_size: Optional[int] :param part_size: Size, in Bytes, of parts that files will be downloaded or uploaded in. :type use_ssl: boolean :param use_ssl: Whether or not to use SSL. By default, SSL is used. Note that not all services support non-ssl connections. :type verify: Optional[boolean/string] :param verify: Whether or not to verify SSL certificates. By default SSL certificates are verified. You can provide the following values: * False - do not validate SSL certificates. SSL will still be used (unless use_ssl is False), but SSL certificates will not be verified. * path/to/cert/bundle.pem - A filename of the CA cert bundle to use. Specify this argument if you want to use a custom CA cert bundle instead of the default one on your system. N)Z ca_filepathFr-) bootstrapregionZcredential_provider part_sizetls_modetls_connection_optionsZthroughput_target_gbps)rrrr ZENABLEDZDISABLEDr Z&override_default_trust_store_from_pathZ verify_peerrZnew_connection_optionsrrZ new_delegaterr )r/r!Z num_threadsZtarget_throughputr0Zuse_sslverifyZevent_loop_groupZ host_resolverr.Zproviderr2r1Ztls_ctx_optionsZclient_tls_optionZcredentails_provider_adapterZ target_gbpsr"r"r#create_s3_crt_clientBsB/  r4c@seZdZdddZddZddZddd Zdd d Zd d d Zd!ddZ ddZ ddZ ddZ d"ddZ ddZddZdS)#CRTTransferManagerNcCs@|dkrt|_||_t||j|_g|_td|_d|_ dS)aFA transfer manager interface for Amazon S3 on CRT s3 client. :type crt_s3_client: awscrt.s3.S3Client :param crt_s3_client: The CRT s3 client, handling all the HTTP requests and functions under then hood :type crt_request_serializer: s3transfer.crt.BaseCRTRequestSerializer :param crt_request_serializer: Serializer, generates unsigned crt HTTP request. :type osutil: s3transfer.utils.OSUtils :param osutil: OSUtils object to use for os-related behavior when using with transfer manager. Nr) r_osutil_crt_s3_clientS3ClientArgsCreator_s3_args_creator_future_coordinatorsr Semaphore _semaphore _id_counter)r Z crt_s3_clientcrt_request_serializerosutilr"r"r#r$s zCRTTransferManager.__init__cCs|Srr"r r"r"r# __enter__szCRTTransferManager.__enter__cGsd}|r d}||dS)NFT _shutdown)r exc_type exc_valueargscancelr"r"r#__exit__szCRTTransferManager.__exit__cCs6|dkr i}|dkri}t|||||d}|d|S)Nbucketkeyfileobj extra_args subscribersZ get_objectr_submit_transfer)r rKrLrMrNrOcallargsr"r"r#downloadszCRTTransferManager.downloadcCs6|dkr i}|dkri}t|||||d}|d|S)NrJZ put_objectrP)r rMrKrLrNrOrRr"r"r#uploadszCRTTransferManager.uploadcCs4|dkr i}|dkri}t||||d}|d|S)N)rKrLrNrOZ delete_objectrP)r rKrLrNrOrRr"r"r#deleteszCRTTransferManager.deleteFcCs||dSrrCr rHr"r"r#shutdownszCRTTransferManager.shutdowncCs |jD]}|s|qdSr)r;donerHr coordinatorr"r"r#_cancel_transferss z$CRTTransferManager._cancel_transferscCs|jD] }|qdSr)r;resultrYr"r"r#_finish_transferss z$CRTTransferManager._finish_transferscCs|jD] }|qdSr)r;%wait_until_on_done_callbacks_completerYr"r"r#_wait_transfers_dones z'CRTTransferManager._wait_transfers_donecCs\|r |z@z |Wn.tk r6|Yntk rHYnXW5|XdSr)r[r_r]KeyboardInterrupt ExceptionrVr"r"r#rDs   zCRTTransferManager._shutdowncKs|jdSr)r=releaser kwargsr"r"r#_release_semaphoresz%CRTTransferManager._release_semaphorec Cs|jg}t|jd}t|j||d}tf|}t|}||zD|j|j |d}||j |||||} |j j f| } WnJtk r} z,|| d|j j |d|d} | | dW5d} ~ XYn X|| |j||jd7_|S) N) transfer_id)metarZZqueuedTrX)after_subscribers)error)reCRTTransferCoordinatorr>CRTTransferMetaCRTTransferFutureAfterDoneHandlerappendr=acquirer:get_crt_callbackget_make_request_argsr8Z make_requestra set_exceptionset_s3_requestr;) r request_type call_argson_done_after_callsrZ componentsfutureZ afterdoneZ on_queuedZ crt_callargsZcrt_s3_requesteon_doner"r"r#rQsF        z#CRTTransferManager._submit_transfer)N)NN)NN)NN)F)F)r)r*r+r$rBrIrSrTrUrWr[r]r_rDrerQr"r"r"r#r5s     r5c@s>eZdZdZd ddZeddZeddZed d ZdS) rlz*Holds metadata about the CRTTransferFutureNcCs||_||_i|_dSr) _transfer_id _call_args _user_context)r rfrvr"r"r#r$1szCRTTransferMeta.__init__cCs|jSr)r}rAr"r"r#rv6szCRTTransferMeta.call_argscCs|jSr)r|rAr"r"r#rf:szCRTTransferMeta.transfer_idcCs|jSr)r~rAr"r"r# user_context>szCRTTransferMeta.user_context)NN) r)r*r+__doc__r$propertyrvrfrr"r"r"r#rl.s   rlc@sDeZdZdddZeddZddZddd Zd d Zd d Z dS)rmNcCs ||_|dkrt|_||_dS)aZThe future associated to a submitted transfer request via CRT S3 client :type meta: s3transfer.crt.CRTTransferMeta :param meta: The metadata associated to the transfer future. :type coordinator: s3transfer.crt.CRTTransferCoordinator :param coordinator: The coordinator associated to the transfer future. N)_metarl _coordinator)r rgrZr"r"r#r$Ds zCRTTransferFuture.__init__cCs|jSr)rrAr"r"r#rgRszCRTTransferFuture.metacCs |jSr)rrXrAr"r"r#rXVszCRTTransferFuture.donecCs|j|dSr)rr\r timeoutr"r"r#r\YszCRTTransferFuture.resultcCs|jdSr)rrHrAr"r"r#rH\szCRTTransferFuture.cancelcCs$|std|jj|dddS)z!Sets the exception on the future.z?set_exception can only be called once the transfer is complete.T)overrideN)rXrrrs)r exceptionr"r"r#rs_s zCRTTransferFuture.set_exception)NN)N) r)r*r+r$rrgrXr\rHrsr"r"r"r#rmCs   rmc@seZdZddZdS)BaseCRTRequestSerializercCs tddS)alSerialize CRT HTTP requests. :type transfer_type: string :param transfer_type: the type of transfer made, e.g 'put_object', 'get_object', 'delete_object' :type future: s3transfer.crt.CRTTransferFuture :rtype: awscrt.http.HttpRequest :returns: An unsigned HTTP request to be used for the CRT S3 client zserialize_http_request()N)NotImplementedError)r transfer_typeryr"r"r#serialize_http_requestjs z/BaseCRTRequestSerializer.serialize_http_requestN)r)r*r+rr"r"r"r#risrc@sVeZdZdddZddZddZdd Zd d Zd d ZddZ ddZ ddZ dS)BotocoreCRTRequestSerializerNcCsl||_|dkri}||||jf||_|jjjd|j|jjjd|j|jjjd|j dS)aSerialize CRT HTTP request using botocore logic It also takes into account configuration from both the session and any keyword arguments that could be passed to `Session.create_client()` when serializing the request. :type session: botocore.session.Session :type client_kwargs: Optional[Dict[str, str]]) :param client_kwargs: The kwargs for the botocore s3 client initialization. Nzrequest-created.s3.*zafter-call.s3.*zbefore-send.s3.*) _session_resolve_client_configZ create_client_clientrgeventsregister_capture_http_request+_change_response_to_serialized_http_request_make_fake_http_response)r session client_kwargsr"r"r#r$zs"     z%BotocoreCRTRequestSerializer.__init__cCsPd}|r|}d|kr$|d}ttd}|r<||}||d<d|d<dS)Nconfig)Zsignature_versionZs3Z service_name)Zget_default_client_configrr merge)r rrZuser_provided_configZ client_configr"r"r#rs  z3BotocoreCRTRequestSerializer._resolve_client_configc Cst|j}|j}|jr&|d|j}g}|jD]6\}}t|trV|||fq4||t|dfq4t j |}d}|j rt |j dr|j }n t|j }t j j|j|||d} | S)N?zutf-8seek)methodpathheadersZ body_stream)rurlrqueryritems isinstancestrroawscrthttpZ HttpHeadersbodyhasattrrZ HttpRequestr) r Z aws_request url_partsZcrt_pathZ headers_listnamevalueZ crt_headersZcrt_body_stream crt_requestr"r"r#_crt_request_from_aws_requests,     z:BotocoreCRTRequestSerializer._crt_request_from_aws_requestcCsT||}|jddkr4t|j}|jd|j|jddk rP|jd|S)Nhostz Content-MD5)rrgetrrsetnetlocremove)r botocore_http_requestrrr"r"r#_convert_to_crt_http_requests   z9BotocoreCRTRequestSerializer._convert_to_crt_http_requestcKs||jd<dS)N http_request)contextr requestrdr"r"r#rsz2BotocoreCRTRequestSerializer._capture_http_requestcKs|d}||d<dS)Nr HTTPRequest)prepare)r rparsedrdrr"r"r#rszHBotocoreCRTRequestSerializer._change_response_to_serialized_http_requestcKstjdditdS)N)botocoreZ awsrequestZ AWSResponseFakeRawResponserr"r"r#rs z5BotocoreCRTRequestSerializer._make_fake_http_responsecCs&t|j|f|j|jd|jdS)N)ZBucketZKeyr)getattrrrKrLrN)r Z client_methodrvr"r"r#_get_botocore_http_requests z7BotocoreCRTRequestSerializer._get_botocore_http_requestcCs|||jj}||}|Sr)rrgrvr)r rryrrr"r"r#rs  z3BotocoreCRTRequestSerializer.serialize_http_request)N) r)r*r+r$rrrrrrrrr"r"r"r#rys   rc@seZdZdddZdS)rNccs||}|sq|VqdSr)read)r amtdecode_contentchunkr"r"r#streams zFakeRawResponse.stream)rN)r)r*r+rr"r"r"r#rsrc@sdeZdZdZdddZeddZddZdd d Zdd d Z ddZ dddZ ddZ ddZ dS)rkz-A helper class for managing CRTTransferFutureNcCs0||_||_t|_d|_d|_t|_dSr) rf _s3_requestrrr _exception _crt_futureEvent _done_event)r rf s3_requestr"r"r#r$s  zCRTTransferCoordinator.__init__cCs|jSr)rrAr"r"r#rsz!CRTTransferCoordinator.s3_requestcCs|jdSr)rrrAr"r"r#set_done_callbacks_completesz2CRTTransferCoordinator.set_done_callbacks_completecCs|j|dSr)rwaitrr"r"r#r^sz.invoke_all_callbacksr")r ryrrrhrr"rr#rqOsz$S3ClientArgsCreator.get_crt_callback)NN)r)r*r+r$rrrqr"r"r"r#r9&s (r9c@seZdZddZddZdS)rcCs||_||_||_||_dSr)r_final_filename_temp_filenamer7)r rZZfinal_filenameZ temp_filenamer@r"r"r#r$jszRenameTempFileHandler.__init__c Kst|d}|r|j|jnTz|j|j|jWn<tk rn}z|j|j|j|W5d}~XYnXdS)Nri)r7Z remove_filerZ rename_filerrarrs)r rdrirzr"r"r#r(pszRenameTempFileHandler.__call__Nr)r*r+r$r(r"r"r"r#risrc@seZdZddZddZdS)rncCs ||_dSr)rrYr"r"r#r$szAfterDoneHandler.__init__cKs|jdSr)rrrcr"r"r#r(szAfterDoneHandler.__call__Nrr"r"r"r#rnsrn)8loggingriorZ awscrt.httprZbotocore.awsrequestrZbotocore.sessionZ awscrt.authrrZ awscrt.iorrrrr Z awscrt.s3r r r r Zbotocore.compatrZbotocore.configrZbotocore.exceptionsrZs3transfer.constantsrrZs3transfer.exceptionsrZs3transfer.futuresrrZs3transfer.utilsrrr getLoggerr)loggerrr4r5rlrmrrrrkr9rrnr"r"r"r# sH         V&o 5C