lcdZddlZddlZddlZddlZddlZ ddlZddlZddl Zddl Z n #e $rdZ YnwxYwddlZddlZddlZddlmZejeZdZ dZ dZdZd Zd Zd Zd Zd ZdZGddZ dZ!dZ"dZ#deedddddfdZ$dZ%dZ&Gdde'Z(dZ)Gddej*Z+Gddej*Z,Gdd ej*Z-eedfd!Z.d"Z/ d+d&Z0d#d'fd(Z1d,d)Z2d*Z3dS)-zDImplements file-like objects for reading and writing from/to AWS S3.NT) constantsi iP)s3s3ns3us3aizs3.amazonaws.comi)zs3://my_bucket/my_keyz&s3://my_key:my_secret@my_bucket/my_keyz8s3://my_key:my_secret@my_server:my_port@my_bucket/my_key InvalidRangeceZdZdZdZdZdS)_ClientWrapperaxWraps a client to inject the appropriate keyword args into each method call. The keyword args are a dictionary keyed by the fully qualified method name. For example, S3.Client.create_multipart_upload. See https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#client This wrapper behaves identically to the client otherwise. c"||_||_dSN)clientkwargs)selfrrs -lib/python3.11/site-packages/smart_open/s3.py__init__z_ClientWrapper.__init__Bs  ct|j|}|jd|zi}t j|fi|S)Nz S3.Client.%s)getattrrrget functoolspartial)r method_namemethodrs r __getattr__z_ClientWrapper.__getattr__FsFk22+!=rBB 226222rN)__name__ __module__ __qualname____doc__rrrrr r 8s<33333rr c tj|}|jtvsJt }t }d}d\}}|j|jz}d|vrNd| ddvr1| dd\}}| d\}}| dd\} } d| vrGd| vrCd}| d\} } | dd\}}t|}n!d| vrd}| d\}} n| } t|j| | ||||| S) NFNN@:r/T)scheme bucket_idkey_idporthostordinary_calling_format access_id access_secret) smart_openutils safe_urlsplitr(SCHEMES DEFAULT_PORT DEFAULT_HOSTnetlocpathsplitintdict) uri_as_string split_urir+r,r-r.r/uriauthheadr* host_portr)s r parse_urirALsw ..}==I  w &&&& D D# *I}  Y^ +C cz3cSYYs^^A..3IIc1%% c#'::c?? =99S!$$LD& d{ sd{ "&#zz# 9__S!,, d4yy "&**S//ii    7#    rcZtfd}d}|B|ds|dr2td|ddn@|dr8|dr0||d|d|dd|B|d t kr1td |d n6|d t kr%|d |z |d |fS)a`Consolidates the parsed Uri with the additional parameters. This is necessary because the user can pass some of the parameters can in two different ways: 1) Via the URI itself 2) Via the transport parameters These are not mutually exclusive, but we have to pick one over the other in a sensible way in order to proceed. c d}n#t$r ix}d<YnwxYw |d}n#t$r ix}|d<YnwxYw|jdi|dS)N client_kwargs S3.Clientr!)KeyErrorupdate)rrD init_kwargstransport_paramss rinjectz#_consolidate_params..injects C,_=MM C C C@B BM,_=== C :' 4KK : : :79 9K- 444 :  $$V$$$$$s   -AArNr.r/zignoring credentials parsed from URL because they conflict with transport_params["client"]. Set transport_params["client"] to None to suppress this warning.)r.r/)aws_access_key_idaws_secret_access_keyr,zignoring endpoint_url parsed from URL because they conflict with transport_params["client"]. Set transport_params["client"] to None to suppress this warning.)r,zhttps://%(host)s:%(port)d) endpoint_url)r:rloggerwarningrGr5)r=rIrJrs ` r_consolidate_paramsrPs,-- % % % % % ! !( + +F 7s;/ 733G 7 (   T 6666 k 7s?37!+."%o"6     T 666 c&k\9  (     V $7#=>>>>     rc6d}fd|D}|r%d}d|d|d}tj|tt|}t |\}t jt}t|d|d|fi|S) N)multipart_upload_kwargs object_kwargsresourceresource_kwargssessionsinglepart_upload_kwargscg|]}|v| Sr!r!).0krIs r zopen_uri..s$???a.>)>????rz^https://github.com/RaRe-Technologies/smart_open/blob/develop/MIGRATING_FROM_OLDER_VERSIONS.rstz8ignoring the following deprecated transport parameters: z. See for detailsr)r*) warningswarn UserWarningrArPr0r1 check_kwargsopen) r=moderI deprecateddetecteddoc_urlmessage parsed_urirs ` ropen_urirgsJ@???:???H, 0 '/hh 9   g{+++3J#6zCS#T#T J   * *41A B BF  ;'H)=t N Nv N NNrFc tdt|tjvrt d|dtj|tjkr|td|tjkrt||||||| } nJ|tjkr-|rt||||| | } n"t|||| | } n Jd |z|| _ | S) a'Open an S3 object for reading or writing. Parameters ---------- bucket_id: str The name of the bucket this object resides in. key_id: str The name of the key within the bucket. mode: str The mode for opening the object. Must be either "rb" or "wb". buffer_size: int, optional The buffer size to use when performing I/O. min_part_size: int, optional The minimum part size for multipart uploads. For writing only. multipart_upload: bool, optional Default: `True` If set to `True`, will use multipart upload for writing to S3. If set to `False`, S3 upload will use the S3 Single-Part Upload API, which is more ideal for small file sizes. For writing only. version_id: str, optional Version of the object, used when reading object. If None, will fetch the most recent version. defer_seek: boolean, optional Default: `False` If set to `True` on a file opened for reading, GetObject will not be called until the first seek() or read(). Avoids redundant API queries when seeking before reading. client: object, optional The S3 client to use when working with boto3. If you don't specify this, then smart_open will create a new client for you. client_kwargs: dict, optional Additional parameters to pass to the relevant functions of the client. The keys are fully qualified method names, e.g. `S3.Client.create_multipart_upload`. The values are kwargs to pass to that method each time it is called. writebuffer: IO[bytes], optional By default, this module will buffer data in memory using io.BytesIO when writing. Pass another binary IO instance here to use it instead. For example, you may pass a file object to buffer to local disk instead of in RAM. Use this to keep RAM usage low at the expense of additional disk IO. If you pass in an open file, then you are responsible for cleaning it up after writing completes. z%rz bad mode: z expected one of Nz$version_id must be None when writing) version_id buffer_size defer_seekrrD) min_part_sizerrD writebuffer)rrDrmFzunexpected mode: %r) rNdebuglocalsr BINARY_MODESNotImplementedError WRITE_BINARY ValueError READ_BINARYReaderMultipartWriterSinglepartWritername) r)r*rarirjrlmultipart_uploadrkrrDrmfileobjs rr`r`s<p LLvxx    9))f!!tttYMcMc"deee &&AZA?@@@ y$$3  !#!'    ' '3  %++' GG'+' GG 3+d2222GL Nrc  |r|||||S||||S#tjj$r*}t d|d|d|d|}||_||d}~wwxYw)N)BucketKey VersionIdRange)r|r}rzunable to access bucket: z key: z version: z error: ) get_objectbotocorer ClientErrorIOError backend_error)rbucketkeyversion range_stringerror wrapped_errors r_getrEs '  Q$$FwVb$cc c$$F<$PP P ? &'''WWWee   ', #&'s55A3 %A..A3cX |jjdS#ttf$rYdSwxYw)zDGiven an IOError from _get, return the 'Error' dictionary from boto.ErrorN)rresponseAttributeErrorrF)ioes r_unwrap_ioerrorrUs? )'22 H %tts ))cFeZdZdZ d dZejfdZd dZd dZ dZ dS) _SeekableRawReaderzHRead an S3 object. This class is internal to the S3 submodule. Nch||_||_||_||_d|_d|_d|_dS)Nr)_client_bucket_key _version_id_content_length _position_body)rrrrris rrz_SeekableRawReader.__init__cs;   %# rc4|tjvrtdtjz|j|jd|_d}d}|tjkrt d|}n:|tjkrt d||jz}nt d| }|j d}n|||j krd}n |dkrd}nd}|r%tj |_|j |_n| |||jS)zSeek to the specified position. :param int offset: The offset in bytes. :param int whence: Where the offset is from. :returns: the position after seeking. :rtype: int z"invalid whence, expected one of %rNrFT) rWHENCE_CHOICESrsrclose WHENCE_STARTmaxWHENCE_CURRENTrrioBytesIO _open_body)roffsetwhencestartstop reached_eofs rseekz_SeekableRawReader.seekss: 1 1 ^AID\\]] ] :  J       Y+ + #6NNEE y/ / #6DN233EEq6'??D   KK  5D,@#@ KK QY KKK  )DJ!1DNN OOE4 ( ( (~rc| ||j}tj||} t |j|j|j|j|}t d||ddtj |d\}}}}||_ ||_|d|_ dS#t$rt}t|}||dt"krt%|dx|_|_ t'j|_ Yd}~dSd}~wwxYw) aOpen a connection to download the specified range of bytes. Store the open file handle in self._body. If no range is specified, start defaults to self._position. start and stop follow the semantics of the http range header, so a stop without a start will read bytes beginning at stop. As a side effect, set self._content_length. Set self._position to self._content_length if start is past end of file. Nz%s: RetryAttempts: %dResponseMetadata RetryAttempts ContentRangeBodyCodeActualObjectSize)rr0r1make_range_stringrrrrrrNrnparse_content_rangerrrrr _OUT_OF_RANGEr9rr) rrrrrunitslengthrerror_responses rrz_SeekableRawReader._open_bodysY  #T #NE!'99%FF  *     H* LL'+,_=    *4)9)M)MhWeNf)g)g &E5$#)D "DN!&)DJJJ- & & &,S11N ););F)C)C})T 47GY8Z4[4[ [DNT1DJJJJJJJ  &s'C E A)D<<Ec h|j||j|jkrdSt gddD]\}} |dkr|j}n|j|}|xjt |z c_|cS#ttj j tj j f$rP}td||||tj||Yd}~d}~wwxYwt#d|||fz)z9Read from the continuous connection with the remote peer.Nr)r&r&rz?%s: caught %r while reading %d bytes, sleeping %ds before retryz-%s: failed to read %d bytes after %d attempts)rrrr enumeratereadlenConnectionResetErrorr exceptions BotoCoreErrorurllib3 HTTPErrorrNrOtimesleepr)rsizeattemptsecondsbinaryerrs rrz_SeekableRawReader.reads` :  OO    >T1 1 3!**:*:*:A > >   GW 2:3!Z__..FF!Z__T22F #f++- %#1", " " " U  7###!!!!!!!! "$EtU\H]]^^^s:B!!+D ADDc(d|jd|jdS)Nzsmart_open.s3._SeekableReader(, )rrrs r__str__z_SeekableRawReader.__str__:>,,, RRrrr#r) rrrr rrrrrrrr!rrrr]s  #,"80000d.*.*.*.*`-_-_-_-_^SSSSSrrc|i}|'|di}tjdi|}|sJt|||_||_||_dS)z~Created the required objects for accessing S3. Ideally, they have been already created for us and we can just reuse them.NrEr)r)rboto3rr rrr)rwrrDrrrHs r_initialize_boto3rso  3#'' R88 22k22 MMM 66BJBJBGGGrceZdZdZdeejdddfdZdZdZ ddZ dd Z d Z dd Z d Zejfd ZdZddZdZdZdZddZddZdZdZdS)ruz]Reads bytes from S3. Implements the io.BufferedIOBase interface of the standard library.NFc <||_||_t|||||t|j|||j|_d|_tj ||_ d|_ ||_ d|_ |s|ddSdS)NrF)r _buffer_sizerrr _raw_reader _current_posr0 bytebuffer ByteBuffer_buffer_eof_line_terminatorrawr) rrrrirjline_terminatorrkrrDs rrzReader.__init__s&'$ vsCCC- L        !,77 DD  /   IIaLLLLL  rcdS)zFlush and close this stream.Nr!rs rrz Reader.closeD rcdS)z+Return True if the stream can be read from.Tr!rs rreadablezReader.readableHtrrc|dkrdS|dkrA||jz}|jj|_|St |j|kr||S|jr|S||||S)z6Read up to size bytes from the object and return them.rr) _read_from_bufferrrrrrrr _fill_buffer)rrouts rrz Reader.readLs 19 3 AX ((**T-=-B-B-D-DDC $ 0 @D J t|   $ 0))$// / 9 ,))++ + $%%d+++rc.||S)zThis is the same as read().)r)rrrs rread1z Reader.read1esyydy###rc|t|}|sdS||dt|<t|S)zLRead up to len(b) bytes into b, and return the number of bytes read.rN)rr)rbdatas rreadintozReader.readintoisGyyQ   1*3t99* 4yyrc|dkrtdtj}|jrt |jdks|j|j}|||xj t |z c_ | |jrn3| |jt |jdk| S)zCRead up to and including the next newline. Returns the bytes read.rz(limits other than -1 not implemented yetr) rqrrrrrreadlinerwriterendswithrgetvalue)rlimitline line_parts rrzReader.readliners B; R%&PQQ Q z||9 $T\!2!2a!7 $ --d.CDDI JJy ! ! !   Y /  !!$"788 $!!###9 $T\!2!2a!7 $}}rcdS)zyIf False, seek(), tell() and truncate() will raise IOError. We offer only seek support, and no truncate support.Tr!rs rseekablezReader.seekable trc|tjkrtj}||jz }|j|||_|j|j|jjk|_ |jS)zSeek to the specified position. :param int offset: The offset in bytes. :param int whence: Where the offset is from. Returns the position after seeking.) rrrrrrremptyrrrrrs rrz Reader.seeksv Y- - (+F d' 'F ,11&&AA %)9)II   rc|jS)z,Return the current position within the file.)rrs rtellz Reader.tell   rctjz Unsupported.rUnsupportedOperationrs rtruncatezReader.truncate %%rctjrrrs rdetachz Reader.detachr rcdS)z Do nothing.Nr!rs r terminatezReader.terminaterrc|s Jd||j|j}|j||jS|SzCreate an **independent** `boto3.s3.Object` instance that points to the same S3 object as this instance. Changes to the returned object will not affect the current instance. z*resource must be a boto3.resource instance)ObjectrrrVersion)rrTobjs rto_boto3zReader.to_boto3sU EEEEEEoodlDI66   ;;t/00 0Jrc|dkr|nt|j}|j|}|xjt|z c_|S)z:Remove at most size bytes from our buffer and return them.r)rrrr)rrparts rrzReader._read_from_buffersRqy7ttc$,&7&7|  && SYY& rcPt||jj}t|j|kro|jsj|j|j}|dkr"td|d|_t|j|kr |jfdSdSdSdS)Nrz$%s: reached EOF while filling bufferT) rr _chunk_sizerrfillrrNrn)rr bytes_reads rrzReader._fill_buffers4122$,$& !ty !**4+;<= 5MB; multipart upload may fail)r|r}UploadIdz the bucket z- does not exist, or is forbidden for access (rr)MIN_MIN_PART_SIZErNrO_min_part_sizerrrrcreate_multipart_upload_retry_if_failed _upload_idrrrrsrr_buf _total_bytes _total_parts_partsr) rrrrlrrDrmrrs rrzMultipartWriter.__init__s  , ,  NN   +$ vsCCC ' 4G /w77 CDOO*   *FFEEE     $ DII#DI s;A==B-B((B-cdSrr!rs rflushzMultipartWriter.flush rc^|jr||jrk|jrdt j|jj|j |j |jd|j i}t|td|n|jr|js Jd|j|j |j |j|j|j |j dtd|d|_dS) NParts)r|r}r$MultipartUploadz%s: completed multipart uploadno multipart upload in progressr|r}r$rr|r}rz-%s: wrote 0 bytes to imitate multipart upload)r*r_upload_next_partr+r)rrrcomplete_multipart_uploadrrr-r(rNrnabort_multipart_upload put_object)rrs rrzMultipartWriter.closes@ 9>>   %  " " $ $ $   P P' 6|I!($+ 6 G W % % % LL94 @ @ @ @ _ P? E E$E E E E L / /|I 0    L # #|I $    LLH$ O O Orc|jduSr)r)rs rclosedzMultipartWriter.closed=s$&&rcdSz+Return True if the stream supports writing.Tr!rs rwritablezMultipartWriter.writableArrcdSzIf False, seek(), tell() and truncate() will raise IOError. We offer only tell support, and no seek or truncate support.Tr!rs rrzMultipartWriter.seekableErrctjrrrs rrzMultipartWriter.seekKr rctjrrrs rr zMultipartWriter.truncateOr rc|jSz#Return the current stream position.r+rs rrzMultipartWriter.tellSrrc*tjdNzdetach() not supportedrrs rr zMultipartWriter.detachZ%&>???rc|j|}|xj|z c_|j|jkr||S)aJWrite the given buffer (bytes, bytearray, memoryview or any buffer interface implementation) to the S3 file. For more information about buffers, see https://docs.python.org/3/c-api/buffer.html There's buffering happening under the covers, so this may not actually do any HTTP transfer right away.)r*rr+rr&r7rrrs rrzMultipartWriter.write]s^## V# 9>>  t2 2 %  " " $ $ $ rc|js Jd|j|j|j|jd|_dS)z'Cancel the underlying multipart upload.r4r5N)r)rr9rrrs rrzMultipartWriter.terminatensTAA AAAA ++< _ ,   rcZ|s Jd||j|jSr)rrr)rrTs rrzMultipartWriter.to_boto3xs1 EEEEEEt|TY777rc X|jdz}td|||j|jdz |jdttj |j j |j |j |j||j}|j|d|dtd|||xjdz c_|jd|jddS) Nr&z3%s: uploading part_num: %i, %i bytes (total %.3fGB)gAr)r|r}r$ PartNumberrETag)rPrOz#%s: upload of part_num #%i finished)r,rNinfor*rr+rr(rrr upload_partrrr)r-appendrnr )rpart_numuploads rr7z!MultipartWriter._upload_next_parts"$q( A   INN     +     q"   (|I#Y        F6N(KKLLL :D(KKK Q q 1rc|Srr!rs r __enter__zMultipartWriter.__enter__ rc^||dS|dSrrrrexc_typeexc_valexc_tbs r__exit__zMultipartWriter.__exit__1   NN      JJLLLLLrc(d|jd|jdS)Nzsmart_open.s3.MultipartWriter(rrrrs rrzMultipartWriter.__str__rrc8d|jd|jd|jdS)Nz%smart_open.s3.MultipartWriter(bucket=rz, min_part_size=r)rrr&rs rrzMultipartWriter.__repr__s, LLL III      rr)rrrr DEFAULT_MIN_PART_SIZErr/rpropertyr<r?rrrrr rr rrrr7rWr_rrr!rrrvrvshKK,****X   """H''X' #,"8&&&&&&&&!!!@@@"888"""H SSS     rrvceZdZdZ ddZdZdZedZdZ dZ e j fd Z dd Zd Zd Zd ZdZdZdZdZdZdS)rwaWrites bytes to S3 using the single part API. Implements the io.BufferedIOBase interface of the standard library. This class buffers all of its input in memory until its `close` method is called. Only then will the data be written to S3 and the buffer is released.Nc$t||||| |j|n/#tjj$r}t d|z|d}~wwxYw|tj|_ n||_ d|_ d|_ dS)N)r|8the bucket %r does not exist, or is forbidden for accessr) rr head_bucketrrrrsrrr*r+r)rrrrrDrmes rrzSinglepartWriter.__init__s $ vsCCC i L $ $F $ 3 3 3 3* i i iWZ``aagh h i  $ DII#DI s1AAAcdSrr!rs rr/zSinglepartWriter.flushr0rcT|jdS|jd |j|j|j|jn4#t jj$r}td|jz|d}~wwxYwt d|d|_dS)Nrr6rgz%s: direct upload finished) r*rrr:rrrrrrsrNrn)rris rrzSinglepartWriter.closes 9  F q b L # #|IY $     * b b bJT\Y[[`a b b  14888 s,AB&A>>Bc|jduSr)r*rs rr<zSinglepartWriter.closedsyD  rcdSr>r!rs rr?zSinglepartWriter.writablerrcdSrAr!rs rrzSinglepartWriter.seekablerrctjrrrs rrzSinglepartWriter.seekr rctjrrrs rr zSinglepartWriter.truncate r rc|jSrErFrs rrzSinglepartWriter.tell rrc*tjdrHrrs rr zSinglepartWriter.detachrIrcZ|j|}|xj|z c_|S)a(Write the given buffer (bytes, bytearray, memoryview or any buffer interface implementation) into the buffer. Content of the buffer will be written to S3 on close as a single-part upload. For more information about buffers, see https://docs.python.org/3/c-api/buffer.html)r*rr+rKs rrzSinglepartWriter.writes0## V# rcdS)z)Nothing to cancel in single-part uploads.Nr!rs rrzSinglepartWriter.terminate"src|Srr!rs rrWzSinglepartWriter.__enter__)rXrc^||dS|dSrrZr[s rr_zSinglepartWriter.__exit__,r`rc<d|jjd|jjdS)Nzsmart_open.s3.SinglepartWriter(rr)_object bucket_namerrs rrzSinglepartWriter.__str__2s';?<;S;S;SUYUaUeUeUeffrc(d|jd|jdS)Nz&smart_open.s3.SinglepartWriter(bucket=rrrrs rrzSinglepartWriter.__repr__5sFJlllTXT]T]T]^^r)NNNr)rrrr rr/rrdr<r?rrrrr rr rrrWr_rrr!rrrwrwsG== 6   &!!X! #,"8&&&&&&&&!!!@@@    ggg_____rrwcB|tjjf}t|D]P} |cS#|$r:td||z dz zt j|YMwxYwtdtd|z)NzwUnable to connect to the endpoint. Check your network connection. Sleeping and retrying %d more times before giving up.r&z-Unable to connect to the endpoint. Giving up.z3Unable to connect to the endpoint after %d attempts) rrEndpointConnectionErrorrangerNcriticalrrr)rattempts sleep_secondsrrs rr(r(9s E)AD ?? X X &799    & & & OO$'/''9A'=?    J} % % % % %  & GHHHKhVWWWs 3>> # get all JSON files under "mybucket/foo/" >>> for key, content in iter_bucket( ... bucket_name, prefix='foo/', ... accept_key=lambda key: key.endswith('.json')): ... print key, len(content) >>> # limit to 10k files, using 32 parallel workers (default is 16) >>> for key, content in iter_bucket(bucket_name, key_limit=10000, workers=32): ... print key, len(content) N)rr)prefix accept_key)ryretries) processesTirz,yielding key #%i: %s, size %i (total %.1fMB)g0Ar&z processed %i keys, total size %i)rrxr _list_bucketrr _download_keyr0 concurrency create_poolimap_unorderedrrNrQr)ryrr key_limitworkersrsession_kwargs total_sizekey_no key_iterator download_keypoolresult_iteratorrcontents r iter_bucketrSsv!  !&      J  L $  L   + +g + > > $--lLII&/&@&@   "FNS'  BCWzK/Gw,    #g,, &J !y)@                 KK2fqj*5MMNNNNNs ""4BDD DcdSrr!)rZs rrsTrc+jKtjjdi|}|d}d} |rt |||}nt ||}|jdi|} |d} | D]} | d} || r| Vn#t $rYnwxYw|dd}|sdS) NrT)r|PrefixContinuationToken)r|rContentsr}NextContinuationTokenr!)rrVSessionrr:list_objects_v2rFr) ryrrrrVrctokenrrrcrs rrrs m#55n55G ^^D ! !F F  =VvVVVFFV<<>r)rNNrr)Nr)4r rrloggingrr\rbotocore.clientrbotocore.exceptionsurllib3.exceptionsr ImportError MISSING_DEPSsmart_open.bytebufferr0smart_open.concurrencysmart_open.utilsr getLoggerrrNrcr%r3r4r5r  URI_EXAMPLES_UPLOAD_ATTEMPTS_SLEEP_SECONDSrr rArPrgr`rrobjectrrBufferedIOBaserurvrwr(rrrrrr!rrrsnKJ  LLLLLL  8 $ $$8!/ % !    33333333(999x6!6!6!rOOOL#' ____D ' ' ' hShShShShShShShSV    G G G G G R G G G TV V V V V b'V V V r{_{_{_{_{_r({_{_{_@"$ XXXX,  aOaOaOaOL!>>++++6s )33