
    Lg3:                        S r SSKJrJr  SSKJrJrJrJrJ	r	J
r
JrJrJrJrJr  \r/ r SSKJr   SSKJr   SSKJr   SS
KJrJrJrJrJrJ r J!r!J"r"J#r#J$r$  SSK%r%\%RL                  " \"5        S r'S r(SS.S jr)SSS.S jjr* SSSS.S jjr+ " S S\5      r,g! \ a    \R+                  S5         Nf = f! \ a    \R+                  S5         Nf = f! \ a    \R+                  S	5         Nf = f! \ a    \R+                  S5         Nf = f)zO
FileSystem abstraction to interact with various local and remote filesystems.
    )_is_path_like_stringify_path)FileSelectorFileTypeFileInfo
FileSystemLocalFileSystemSubTreeFileSystem_MockFileSystemFileSystemHandlerPyFileSystem_copy_files_copy_files_selector)AzureFileSystemr   )HadoopFileSystemr   )GcsFileSystemr   )
AwsDefaultS3RetryStrategyAwsStandardS3RetryStrategyS3FileSystem
S3LogLevelS3RetryStrategyensure_s3_initializedfinalize_s3ensure_s3_finalizedinitialize_s3resolve_s3_regionNr   c                 ~    U [         ;   a  [        SR                  U 5      5      e[        SR                  U 5      5      e)Nz<The pyarrow installation is not built with support for '{0}'z*module 'pyarrow.fs' has no attribute '{0}')_not_importedImportErrorformatAttributeError)names    *lib/python3.12/site-packages/pyarrow/fs.py__getattr__r$   I   sB    }F4L
 	

 4;;DA     c                 V   [         R                  " U 5      u  pUR                  U5      nU(       ax  UR                  U/5      S   nUR                  [
        R                  :w  a:  [        SR                  UR                  R                  UR                  U 5      5      e[        X!5      nU$ )Nr   zThe path component of the filesystem URI must point to a directory but it has a type: `{}`. The path component is `{}` and the given filesystem URI is `{}`)r   from_urinormalize_pathget_file_infotyper   	Directory
ValueErrorr    r"   pathr
   )uri
filesystemprefixprefix_infos       r#   _filesystem_from_strr2   U   s     $,,S1J&&v.F ..x8;x111??Ev$$));+;+;S@  'v:
r%   Fuse_mmapc                   [        U [        5      (       a  U $ [        U [        5      (       a  U(       a  [        S5      e[	        U 5      $  SS Kn[        XR                  5      (       a6  [        U 5      R                  S:X  a	  [        US9$ [        [        U 5      5      $  [        SR                  [        U 5      5      5      e! [         a     N/f = f)NzXSpecifying to use memory mapping not supported for filesystem specified as an URI stringr   r	   r3   zlUnrecognized filesystem: {}. `filesystem` argument must be a FileSystem instance or a valid file system URI')
isinstancer   strr,   r2   fsspecAbstractFileSystemr*   __name__r	   r   FSSpecHandlerr   	TypeErrorr    )r/   r4   r8   s      r#   _ensure_filesystemr=   i   s    *j))	J	$	$8  $J//	; j";";<<J((,==&99j 9::	 = 	::@&;   s   C 
CC)
memory_mapc                   [        U 5      (       d  Ub  [        S5      eX4$ Ub^  [        XS9n[        U[        5      (       a  [        U 5      n O [        U [        5      (       d  [        S5      eUR                  U 5      n X4$ [        U 5      n [	        US9n UR                  U 5      nUR                  [        R                  :g  nU(       d   [        R                  " U 5      u  pX4$ UR                  U 5      n X4$ ! [         a    SnSn NGf = f! [         a+  nS[        U5      ;  a  S[        U5      ;  a  e  SnAX4$ SnAff = f)zT
Return filesystem/path from path which could be an URI or a plain
filesystem path.
Nzg'filesystem' passed but the specified path is file-like, so there is nothing to open with 'filesystem'.r3   zPExpected string path; path-like objects are only allowed with a local filesystemFzempty schemezCannot parse URI)r   r,   r=   r6   r	   r   r7   r<   r(   r)   r*   r   NotFoundr   r'   )r-   r/   r>   	file_infoexists_locallyes         r#   _resolve_filesystem_and_pathrD      sl   
 !?  '
H
j/22"4(DD#&&*  ((.4 D
 !*5J?,,T2	
 $..H,=,== 	)2248J  ((.+  	  	 SV+*#a&8 	s*   C= D =DD
EEEi   T)
chunk_sizeuse_threadsc                    [        X5      u  pg[        X5      u  pUR                  U5      n
U
R                  [        R                  :X  a  [        USS9n[        XkXXE5        g[        XgXXE5        g)a  
Copy files between FileSystems.

This functions allows you to recursively copy directories of files from
one file system to another, such as from S3 to your local machine.

Parameters
----------
source : string
    Source file path or URI to a single file or directory.
    If a directory, files will be copied recursively from this path.
destination : string
    Destination file path or URI. If `source` is a file, `destination`
    is also interpreted as the destination file (not directory).
    Directories will be created as necessary.
source_filesystem : FileSystem, optional
    Source filesystem, needs to be specified if `source` is not a URI,
    otherwise inferred.
destination_filesystem : FileSystem, optional
    Destination filesystem, needs to be specified if `destination` is not
    a URI, otherwise inferred.
chunk_size : int, default 1MB
    The maximum size of block to read before flushing to the
    destination file. A larger chunk_size will use more memory while
    copying but may help accommodate high latency FileSystems.
use_threads : bool, default True
    Whether to use multiple threads to accelerate copying.

Examples
--------
Inspect an S3 bucket's files:

>>> s3, path = fs.FileSystem.from_uri(
...            "s3://registry.opendata.aws/roda/ndjson/")
>>> selector = fs.FileSelector(path)
>>> s3.get_file_info(selector)
[<FileInfo for 'registry.opendata.aws/roda/ndjson/index.ndjson':...]

Copy one file from S3 bucket to a local directory:

>>> fs.copy_files("s3://registry.opendata.aws/roda/ndjson/index.ndjson",
...               "file:///{}/index_copy.ndjson".format(local_path))

>>> fs.LocalFileSystem().get_file_info(str(local_path)+
...                                    '/index_copy.ndjson')
<FileInfo for '.../index_copy.ndjson': type=FileType.File, size=...>

Copy file using a FileSystem object:

>>> fs.copy_files("registry.opendata.aws/roda/ndjson/index.ndjson",
...               "file:///{}/index_copy.ndjson".format(local_path),
...               source_filesystem=fs.S3FileSystem())
T	recursiveN)rD   r)   r*   r   r+   r   r   r   )sourcedestinationsource_filesystemdestination_filesystemrE   rF   	source_fssource_pathdestination_fsdestination_pathrA   
source_sels               r#   
copy_filesrS      sz    p :I (D($N ''4I~~+++!+>
Y+'	6 	I"	-r%   c                       \ rS rSrSrS rS rS rS rS r	\
S 5       rS	 rS
 rS rS rS rS rS rS rS rS rS rS rS rS rSrg)r;   i  z
Handler for fsspec-based Python filesystems.

https://filesystem-spec.readthedocs.io/en/latest/index.html

Parameters
----------
fs : FSSpec-compliant filesystem instance

Examples
--------
>>> PyFileSystem(FSSpecHandler(fsspec_fs)) # doctest: +SKIP
c                     Xl         g Nfs)selfrX   s     r#   __init__FSSpecHandler.__init__  s    r%   c                 j    [        U[        5      (       a  U R                  UR                  :H  $ [        $ rV   r6   r;   rX   NotImplementedrY   others     r#   __eq__FSSpecHandler.__eq__  (    e]++77ehh&&r%   c                 j    [        U[        5      (       a  U R                  UR                  :g  $ [        $ rV   r]   r_   s     r#   __ne__FSSpecHandler.__ne__#  rc   r%   c                     U R                   R                  n[        U[        5      (       a  US   nSR	                  U5      $ )Nr   z
fsspec+{0})rX   protocolr6   listr    )rY   rh   s     r#   get_type_nameFSSpecHandler.get_type_name(  s8    77##h%%{H""8,,r%   c                     U$ rV    rY   r-   s     r#   r(   FSSpecHandler.normalize_path.  s    r%   c           	          US   nUS   S:X  a  [         R                  nO,US   S:X  a  [         R                  nS nO[         R                  n[	        XX!R                  SS 5      S9$ )Nsizer*   file	directorymtime)rq   rt   )r   Filer+   Unknownr   get)r-   inforq   ftypes       r#   _create_file_infoFSSpecHandler._create_file_info1  s_    F|<6!MME&\[(&&ED$$E$hhw6MNNr%   c                    / nU H?  n U R                   R                  U5      nUR                  U R                  X45      5        MA     U$ ! [         a-    UR                  [        U[        R                  5      5         M{  f = frV   )rX   rx   appendrz   FileNotFoundErrorr   r   r@   )rY   pathsinfosr-   rx   s        r#   r)   FSSpecHandler.get_file_info>  su    DAww||D) T33D?@  	 % @XdH,=,=>?@s   A

3B Bc                 |   U R                   R                  UR                  5      (       dg  U R                   R                  UR                  5      (       a  [	        UR                  5      eUR
                  (       a  / $ [        UR                  5      eUR                  (       a  S nOSn/ nU R                   R                  UR                  USSS9nUR                  5        HX  u  pVUR                  S5      nUR                  R                  S5      nXx:w  d  M8  UR                  U R                  XV5      5        MZ     U$ )N   T)maxdepthwithdirsdetail/)rX   isdirbase_direxistsNotADirectoryErrorallow_not_foundr~   rI   finditemsstripr}   rz   )	rY   selectorr   r   selected_filesr-   rx   _pathr   s	            r#   get_file_info_selector$FSSpecHandler.get_file_info_selectorI  s   ww}}X..//ww~~h//00():):;;++I+H,=,=>>HH4 & 
 )..0JDJJsOE((..s3H  T33D?@ 1 r%   c                 X     U R                   R                  XS9  g ! [         a     g f = f)N)create_parents)rX   mkdirFileExistsError)rY   r-   rI   s      r#   
create_dirFSSpecHandler.create_dirf  s+    	GGMM$M9 		s    
))c                 8    U R                   R                  USS9  g NTrH   )rX   rmrn   s     r#   
delete_dirFSSpecHandler.delete_dirm  s    

44
(r%   c                 n    U R                   R                  USS9nU H|  nU R                   R                  U5      (       a  U R                   R	                  USS9  M?  U R                   R                  U5      (       d  Ma  U R                   R	                  U5        M~     g ! [         a    U(       a   g e f = f)NF)r   TrH   )rX   listdirr~   r   r   isfile)rY   r-   missing_dir_oksubpathssubpaths        r#   _delete_dir_contents"FSSpecHandler._delete_dir_contentsp  s    	wwtE:H
  Gww}}W%%

7d
3((

7#	  	 ! 		s   B B42B4c                 j    UR                  S5      S:X  a  [        SUS5      eU R                  X5        g )Nr    z$delete_dir_contents called on path '')r   r,   r   )rY   r-   r   s      r#   delete_dir_contents!FSSpecHandler.delete_dir_contents}  s6    ::c?b 6cC C!!$7r%   c                 &    U R                  S5        g )Nr   )r   )rY   s    r#   delete_root_dir_contents&FSSpecHandler.delete_root_dir_contents  s    !!#&r%   c                     U R                   R                  U5      (       d  [        U5      eU R                   R                  U5        g rV   )rX   r   r~   r   rn   s     r#   delete_fileFSSpecHandler.delete_file  s2     ww~~d###D))

4r%   c                 8    U R                   R                  XSS9  g r   )rX   mvrY   srcdests      r#   moveFSSpecHandler.move  s    

3
-r%   c                 :    U R                   R                  X5        g rV   )rX   copyr   s      r#   	copy_fileFSSpecHandler.copy_file  s     	Sr%   c                     SSK Jn  U R                  R                  U5      (       d  [	        U5      eU" U R                  R                  USS9SS9$ Nr   
PythonFilerbmoderpyarrowr   rX   r   r~   openrY   r-   r   s      r#   open_input_streamFSSpecHandler.open_input_stream  A    &ww~~d###D))$'',,t$,7cBBr%   c                     SSK Jn  U R                  R                  U5      (       d  [	        U5      eU" U R                  R                  USS9SS9$ r   r   r   s      r#   open_input_fileFSSpecHandler.open_input_file  r   r%   c                 L    SSK Jn  U" U R                  R                  USS9SS9$ )Nr   r   wbr   wr   r   rX   r   rY   r-   metadatar   s       r#   open_output_stream FSSpecHandler.open_output_stream  #    &$'',,t$,7cBBr%   c                 L    SSK Jn  U" U R                  R                  USS9SS9$ )Nr   r   abr   r   r   r   s       r#   open_append_stream FSSpecHandler.open_append_stream  r   r%   rW   N)r:   
__module____qualname____firstlineno____doc__rZ   ra   re   rj   r(   staticmethodrz   r)   r   r   r   r   r   r   r   r   r   r   r   r   r   __static_attributes__rm   r%   r#   r;   r;     s    

- 
O 
O	:)$8'. CCC
Cr%   r;   rV   )NN)-r   pyarrow.utilr   r   pyarrow._fsr   r   r   r   r	   r
   r   r   r   r   r   	FileStatsr   pyarrow._azurefsr   r   r}   pyarrow._hdfsr   pyarrow._gcsfsr   pyarrow._s3fsr   r   r   r   r   r   r   r   r   r   atexitregisterr$   r2   r=   rD   rS   r;   rm   r%   r#   <module>r      s6  $ 8    	,0-.*,)L L L 
OO'(	( 05 <7e 7v ?CH-&DH-VcC% cCA  ,*+,
  -+,-
  *)*  )()sD   B B7 C C1 B43B47CCC.-C.1D
D