OOIdAddlZddlZddlZddlZddlZddlZddlmZddlm Z ddl m Z ddl m Z ddlmZdZd!dZd!d ZiZd Zd Zd Zd"dZdZdZdZdZd"dZdZdZdZdZ dZ!d#dZ"dZ#dZ$edZ%d$dZ&d Z'dS)%N)contextmanager)partial)md5)version)urlsplitiPctjd|stjd|d|dSt|}|jpd}|jr"d|j|jg}n|j}|dkr.tjd|}|rd|z}|d vr||dS||d}|jr|j d d d  d d d|d<|dvr|d|dz|d<n |d|d<|j r |j |d<|j r |j |d<|j r |j |d<|j r |j |d<|jr |j|d<|rt|||S)aInfer storage options from URL path and merge it with existing storage options. Parameters ---------- urlpath: str or unicode Either local absolute file path or URL (hdfs://namenode:8020/file.csv) inherit_storage_options: dict (optional) Its contents will get merged with the inferred information from the given path Returns ------- Storage options dict. Examples -------- >>> infer_storage_options('/mnt/datasets/test.csv') # doctest: +SKIP {"protocol": "file", "path", "/mnt/datasets/test.csv"} >>> infer_storage_options( ... 'hdfs://username:pwd@node:123/mnt/datasets/test.csv?q=1', ... inherit_storage_options={'extra': 'value'}, ... ) # doctest: +SKIP {"protocol": "hdfs", "username": "username", "password": "pwd", "host": "node", "port": 123, "path": "/mnt/datasets/test.csv", "url_query": "q=1", "extra": "value"} z^[a-zA-Z]:[\\/]z^[a-zA-Z0-9]+://Nfile)protocolpath#z^/([a-zA-Z])[:|]([\\/].*)$z%s:%s)httphttps@:rhost)s3s3agcsgsr portusernamepassword url_query url_fragment)rematchrschemefragmentjoinr groupsnetlocrsplitrrrqueryupdate_storage_options)urlpathinherit_storage_options parsed_pathr r windows_pathoptionss ,lib/python3.11/site-packages/fsspec/utils.pyinfer_storage_optionsr-s< #W--5 8' 1 1 9"G4447##K!+VH xx);+?@AA6x =tDD  3\00222D$$$$g666#T22G7&,33C;;B?FFsANNqQ 1 1 1%fo?GFOO%foGFO   /).GFO   7"-"6GJ    7"-"6GJ 1*0 7"-"6Aw(?@@@ Nc|si}t|t|z}|rC|D]@}||||krtd|zA||dS)Nz;Collision between inferred and specified storage option: %s)setgetKeyErrorupdate)r+ inherited collisions collisions r,r&r&bs  WI.J#  I{{9%%y)A)AAA"$-.B  NN9r.ctj|dd}|t vr t |SdS)aInfer compression, if available, from filename. Infer a named compression type, if registered and available, from filename extension. This includes builtin (gz, bz2, zip) compressions, as well as optional compressions. See fsspec.compression.register_compression. r.N)osr splitextstriplower compressions)filename extensions r,infer_compressionr@tsV  **2.44S99??AAIL  I&&! r.c|dz }ttjtj|fd}|S)aReturns a function that receives a single integer and returns it as a string padded by enough zero characters to align with maximum possible integer >>> name_f = build_name_function(57) >>> name_f(7) '07' >>> name_f(31) '31' >>> build_name_function(1000)(42) '0042' >>> build_name_function(999)(42) '042' >>> build_name_function(0)(0) '0' g:0yE>cHt|SN)strzfill)i pad_lengths r, name_functionz*build_name_function..name_functions1vv||J'''r.)intmathceillog10)max_intrHrGs @r,build_name_functionrNsP& tOGTYtz'223344J((((( r.c|dkrdSd} ||}|sdS|r||zn|} ||vra||}||t ||z z t |zdSt ||krdSn#t t f$rYnwxYw|t | d})aSeek current file to file start, file end, or byte after delimiter seq. Seeks file to next chunk delimiter, where chunks are defined on file start, a delimiting sequence, and file end. Use file.tell() to see location afterwards. Note that file start is a valid split, so must be at offset > 0 to seek for delimiter. Parameters ---------- file: a file delimiter: bytes a delimiter like ``b'\n'`` or message sentinel, matching file .read() type blocksize: int Number of bytes to read from the file at once. Returns ------- Returns True if a delimiter was found, False if at file start or end. rFNT)tellreadindexseeklenOSError ValueError)r delimiter blocksizelastcurrentfullrFs r,seek_delimiterr\s. yy{{au D'))I&& 5!%2tg~~7 D  JJy)) $))++TQ7#i..HIIItW ))u*$    D S^^O%%&!'sA#B<&B<<CCFc|r||t||d}||S|}|||z z}|||zt||d}|}|r|r|t |z}|r|r|t |z}|}||z }||||} | S)aRead a block of bytes from a file Parameters ---------- f: File Open file offset: int Byte offset to start read length: int Number of bytes to read, read through end of file if None delimiter: bytes (optional) Ensure reading starts and stops at delimiter bytestring split_before: bool (optional) Start/stop read *before* delimiter bytestring. If using the ``delimiter=`` keyword argument we ensure that the read starts and stops at delimiter boundaries that follow the locations ``offset`` and ``offset + length``. If ``offset`` is zero then we start at zero, regardless of delimiter. The bytestring returned WILL include the terminating delimiter string. Examples -------- >>> from io import BytesIO # doctest: +SKIP >>> f = BytesIO(b'Alice, 100\nBob, 200\nCharlie, 300') # doctest: +SKIP >>> read_block(f, 0, 13) # doctest: +SKIP b'Alice, 100\nBo' >>> read_block(f, 0, 13, delimiter=b'\n') # doctest: +SKIP b'Alice, 100\nBob, 200\n' >>> read_block(f, 10, 10, delimiter=b'\n') # doctest: +SKIP b'Bob, 200\nCharlie, 300' i)rSr\rQrPrT) foffsetlengthrW split_beforefound_start_delimstartfound_end_delimendbs r, read_blockrgsJ v*1i?? >6688O%&.  uv~(Iu==ffhh  $ $ S^^ #E  "| " 3y>> !CuFF6NNN vA Hr.c8|r||fz } tt|S#t$rEtt|dcYSwxYw)zDeterministic token (modified from dask.base) >>> tokenize([1, 2, '3']) '9d71491b50023b06fc76928e6eddb952' >>> tokenize('Hello') == tokenize('Hello') True F)usedforsecurity)rrDencode hexdigestrV)argskwargss r,tokenizerns  J3t99##%%&&00222 JJJ3t99##%%u===GGIIIIIJs?A A BBct|tr|St|dr|St|tjrt|St|dr|jS|S)a6Attempt to convert a path-like object to a string. Parameters ---------- filepath: object to be converted Returns ------- filepath_str: maybe a string version of the object Notes ----- Objects supporting the fspath protocol are coerced according to its __fspath__ method. For backwards compatibility with older Python version, pathlib.Path objects are specially coerced. Any other object is passed through unchanged, which includes bytes, strings, buffers, or anything else that's not even path-like. __fspath__r ) isinstancerDhasattrrppathlibPathr )filepaths r,stringify_pathrv#s~,(C   < ( (""$$$ Hgl + +8}} 6 " "}r.c>||i|}||SrC)_determine_worker)clsrlrminsts r, make_instancer{Es- 3   D Kr.cd|DtdD}d}t|D]"tfdD}|sn#|z dddS)z;For a list of paths, find the shortest prefix common to allc8g|]}|dS)/)split.0ps r, z!common_prefix..Ms" ) ) )aQWWS\\ ) ) )r.c34K|]}t|VdSrC)rTrs r, z common_prefix..Ns(%%!s1vv%%%%%%r.rc3HK|]}|dkVdS)rN)rrrFpartss r,rz common_prefix..Qs455!!A$%(1+%555555r.r~N)minrangeallr!)pathslmaxrerFrs @@r, common_prefixrKs ) )5 ) ) )E %%u%%% % %D C 4[[55555u55555  E HA 88E!HRaRL ! !!r.cJttri|pd}dt ||rdddfd|Dn"t |t ksJS)aIn bulk file operations, construct a new file tree from a list of files Parameters ---------- paths: list of str The input file tree path2: str or list of str Root to construct the new list in. If this is already a list of str, we just assert it has the right number of elements. is_dir: bool (optional) For the special case where the input in one element, whether to regard the value as the target path, or as a directory to put a file path within. If None, a directory is inferred if the path ends in '/' exists: bool (optional) For a str destination, it is already exists (and is a dir), files should end up inside. Returns ------- list of str r~rrc>g|]}|dS)r)replace)rrcppath2s r,rzother_paths..ts)888Q2ua((888r.)rqrDendswithrstriprr$rT)rris_direxistsrs ` @r, other_pathsrXs,%(.5>>#.. S!! 5 ! !  &3""1%B88888%8885zzSZZ'''' Lr.c,t|tSrC)rq BaseExceptionobjs r, is_exceptionrzs c= ) ))r.c6dD]}t||sdSdS)N)rQcloserPFT)rr)r^attrs r, isfileliker~s4)q$ 55  4r.chtjd|d}t|dkr|dSdS)Nz (\:\:|\://)rrr )rrrT)urlrs r, get_protocolrs2 H^S! , ,E 5zzA~~Qx 6r.cddlm} t|t|ddS#tt f$rYdSwxYw)z*Can the given URL be used with open_local?r)get_filesystem_class local_fileF)fsspecrgetattrrrV ImportError)r rs r, can_be_localrsf++++++++L,>,>??uUUU  $uus&/AAc|tjvr)tj|}t|dr|jS t |S#YnxYw ddl}||}|jS#ttf$rYdSwxYw)aFor given package name, try to find the version without importing it Import and package.__version__ is still the backup here, so an import *might* happen. Returns either the version string, or None if the package or the version was not readily found. __version__rN) sysmodulesrrrr importlib import_modulerAttributeError)namemodrs r,"get_package_version_without_importrs s{k$ 3 & & #? " t}}  %%d++  (ttsAA A00BBDEBUGTcZ||td|ptj|}tj}tjd}|||r|j||| ||S)Nz+Provide either logger object or logger namezD%(asctime)s - %(name)s - %(levelname)s - %(funcName)s -- %(message)s) rVlogging getLogger StreamHandler Formatter setFormatterhandlersclear addHandlersetLevel)logger logger_namelevelrhandle formatters r, setup_loggingrs ~+-FGGG  5w(55F  " $ $F!NI  """   f OOE Mr.c,||SrC)unstrip_protocol)rfss r,_unstrip_protocolrs  t $ $$r.c"fdfd}|S)zqMirror attributes and methods from the given origin_name attribute of the instance to the decorated classcDt|}t||SrC)r)methodselforigin origin_names r, origin_getterz"mirror_from..origin_getters"{++vv&&&r.cnD]0}t|}t||t|1|SrC)rsetattrproperty)ryrwrapped_methodmethodsrs r,wrapperzmirror_from..wrappersC ; ;F$]F;;N C.!9!9 : : : : r.r)rrrrs`` @r, mirror_fromrsC ''''' Nr.c#K|VdSrCrrs r, nullcontextrs IIIIIr.c &t|tstt|ts|gt|z}t|ts|gt|z}t|t|ks t|t|krtt|dkr|||fSd|D}|r3dt t t |||D\}}}|r|dd}|dd}|dd}tdt|D]} || || dz kr |d || || dz ks,|| |dz |ks|g|| |dz |krR||| ||| ||| || |d<|||fS|||fS)a}Merge adjacent byte-offset ranges when the inter-range gap is <= `max_gap`, and when the merged byte range does not exceed `max_block` (if specified). By default, this function will re-order the input paths and byte ranges to ensure sorted order. If the user can guarantee that the inputs are already sorted, passing `sort=False` will skip the re-ordering. rcg|]}|pdS)rr)rss r,rz'merge_offset_ranges..s % % %af1 % % %r.c,g|]}t|Sr)list)rvs r,rz'merge_offset_ranges..s.    GG   r.Nr) rqr TypeErrorrTrVzipsortedrappend) rstartsendsmax_gap max_blocksort new_paths new_startsnew_endsrFs r,merge_offset_rangesrsP eT " " fd # #'CJJ& dD ! !%x#e**$ 6{{c%jj  CIIU$;$; 6{{afd"" % %f % % %F    vt,,   vt /"1"I BQBZ 8q#e**%% ' 'AQx5Q<''HRL,@aE!a%L((AY"-88*Q*R.0HI/U/U  q***!!&),,,Q(((( $Aw *h.. &$ r.c|} |dd||S#||wxYw)z+Find length of any open read-mode file-liker)rPrS)filelikeposs r, file_sizersR --//C}}Q"" c cs AArC)NF)NNrT)rNT)(rrJr9rsrr contextlibr functoolsrhashlibrimportlib.metadatar urllib.parserDEFAULT_BLOCK_SIZEr-r&r=r@rNr\rgrnrvr{rrrrrrrrrrrrrrr.r,rs-  %%%%%%&&&&&&!!!!!!OOOOd      ' ' ':.'.'.'b> > > > BJJJ(D " " "D***4     %%%$<<<<~r.