a
    5PÎjÀ8  ã                   @   s:  d Z ddlmZmZ ddlmZmZmZmZm	Z	m
Z
mZmZmZmZmZ eZg ZzddlmZ W n ey|   e d¡ Y n0 zddlmZ W n eyª   e d¡ Y n0 z ddlmZmZmZmZmZ W n eyè   e d	¡ Y n0 eƒ  d
d„ Zdd„ Zddd„Z ddd„Z!ddddœdd„Z"G dd„ deƒZ#dS )zO
FileSystem abstraction to interact with various local and remote filesystems.
é    )Ú_is_path_likeÚ_stringify_path)ÚFileSelectorÚFileTypeÚFileInfoÚ
FileSystemÚLocalFileSystemÚSubTreeFileSystemÚ_MockFileSystemÚFileSystemHandlerÚPyFileSystemÚ_copy_filesÚ_copy_files_selector)ÚHadoopFileSystemr   )ÚGcsFileSystemr   )ÚS3FileSystemÚ
S3LogLevelÚinitialize_s3Úfinalize_s3Úresolve_s3_regionr   c                 C   s(   | t v rtd | ¡ƒ‚td | ¡ƒ‚d S )Nz<The pyarrow installation is not built with support for '{0}'z*module 'pyarrow.fs' has no attribute '{0}')Ú_not_importedÚImportErrorÚformatÚAttributeError)Úname© r   úZ/mounts/lovelace/software/anaconda3/envs/metadmg/lib/python3.9/site-packages/pyarrow/fs.pyÚ__getattr__?   s    ÿÿÿr   c                 C   s^   t  | ¡\}}| |¡}|rZ| |g¡d }|jtjkrPtd |jj	|j
| ¡ƒ‚t||ƒ}|S )Nr   z›The path component of the filesystem URI must point to a directory but it has a type: `{}`. The path component is `{}` and the given filesystem URI is `{}`)r   Úfrom_uriÚnormalize_pathÚget_file_infoÚtyper   Ú	DirectoryÚ
ValueErrorr   r   Úpathr	   )ÚuriÚ
filesystemÚprefixZprefix_infor   r   r   Ú_filesystem_from_strK   s    
ýÿ
r(   Fc                 C   sÈ   t | tƒr| S t | tƒr,|r$tdƒ‚t| ƒS zdd l}W n tyJ   Y n20 t | |jƒr|t| ƒj	dkrpt
|d�S tt| ƒƒS dd lm} t | |j
ƒržt
|d�S |r²t | |jƒr²| S td t| ƒ¡ƒ‚d S )NzXSpecifying to use memory mapping not supported for filesystem specified as an URI stringr   r   )Úuse_mmapzlUnrecognized filesystem: {}. `filesystem` argument must be a FileSystem instance or a valid file system URI')Ú
isinstancer   Ústrr#   r(   Úfsspecr   ZAbstractFileSystemr!   Ú__name__r   r   ÚFSSpecHandlerZpyarrow.filesystemr&   Ú	TypeErrorr   )r&   r)   Úallow_legacy_filesystemr,   Zlegacyfsr   r   r   Ú_ensure_filesystem_   s4    

ÿ

þÿr1   Nc              
   C   s  t | ƒs |durtdƒ‚|| fS |durpt||d�}t|tƒrHt| ƒ} nt| tƒsZtdƒ‚|sh| | ¡} || fS t| ƒ} tƒ }z| 	| ¡}W n ty¦   d}d}Y n0 |j
tjk}|�szt | ¡\}} W n2 tyþ } zdt|ƒvrê‚ W Y d}~n
d}~0 0 n
| | ¡} || fS )z`
    Return filesystem/path from path which could be an URI or a plain
    filesystem path.
    Nzg'filesystem' passed but the specified path is file-like, so there is nothing to open with 'filesystem'.)r0   zPExpected string path; path-like objects are only allowed with a local filesystemFzempty scheme)r   r#   r1   r*   r   r   r+   r/   r   r    r!   r   ÚNotFoundr   r   )r$   r&   r0   Ú	file_infoZexists_locallyÚer   r   r   Ú_resolve_filesystem_and_pathˆ   sF    ÿÿ


ÿ


r5   i   T)Ú
chunk_sizeÚuse_threadsc                C   sh   t | |ƒ\}}t ||ƒ\}}	| |¡}
|
jtjkrRt|dd�}t||||	||ƒ nt||||	||ƒ dS )a¦  
    Copy files between FileSystems.

    This functions allows you to recursively copy directories of files from
    one file system to another, such as from S3 to your local machine.

    Parameters
    ----------
    source : string
        Source file path or URI to a single file or directory.
        If a directory, files will be copied recursively from this path.
    destination : string
        Destination file path or URI. If `source` is a file, `destination`
        is also interpreted as the destination file (not directory).
        Directories will be created as necessary.
    source_filesystem : FileSystem, optional
        Source filesystem, needs to be specified if `source` is not a URI,
        otherwise inferred.
    destination_filesystem : FileSystem, optional
        Destination filesystem, needs to be specified if `destination` is not
        a URI, otherwise inferred.
    chunk_size : int, default 1MB
        The maximum size of block to read before flushing to the
        destination file. A larger chunk_size will use more memory while
        copying but may help accommodate high latency FileSystems.
    use_threads : bool, default True
        Whether to use multiple threads to accelerate copying.

    Examples
    --------
    Inspect an S3 bucket's files:

    >>> s3, path = fs.FileSystem.from_uri(
    ...            "s3://registry.opendata.aws/roda/ndjson/")
    >>> selector = fs.FileSelector(path)
    >>> s3.get_file_info(selector)
    [<FileInfo for 'registry.opendata.aws/roda/ndjson/index.ndjson':...]

    Copy one file from S3 bucket to a local directory:

    >>> fs.copy_files("s3://registry.opendata.aws/roda/ndjson/index.ndjson",
    ...               "file:///{}/index_copy.ndjson".format(local_path))

    >>> fs.LocalFileSystem().get_file_info(str(local_path)+
    ...                                    '/index_copy.ndjson')
    <FileInfo for '.../index_copy.ndjson': type=FileType.File, size=...>

    Copy file using a FileSystem object:

    >>> fs.copy_files("registry.opendata.aws/roda/ndjson/index.ndjson",
    ...               "file:///{}/index_copy.ndjson".format(local_path),
    ...               source_filesystem=fs.S3FileSystem())
    T©Ú	recursiveN)r5   r    r!   r   r"   r   r   r   )ÚsourceZdestinationZsource_filesystemZdestination_filesystemr6   r7   Z	source_fsÚsource_pathZdestination_fsZdestination_pathr3   Z
source_selr   r   r   Ú
copy_filesÅ   s"    8ÿÿ
þþr<   c                   @   s´   e Zd ZdZdd„ Zdd„ Zdd„ Zdd	„ Zd
d„ Ze	dd„ ƒZ
dd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zd d!„ Zd"d#„ Zd$d%„ Zd&d'„ Zd(d)„ Zd*S )+r.   a   
    Handler for fsspec-based Python filesystems.

    https://filesystem-spec.readthedocs.io/en/latest/index.html

    Parameters
    ----------
    fs : FSSpec-compliant filesystem instance.

    Examples
    --------
    >>> PyFileSystem(FSSpecHandler(fsspec_fs)) # doctest: +SKIP
    c                 C   s
   || _ d S ©N)Úfs)Úselfr>   r   r   r   Ú__init__  s    zFSSpecHandler.__init__c                 C   s   t |tƒr| j|jkS tS r=   ©r*   r.   r>   ÚNotImplemented©r?   Úotherr   r   r   Ú__eq__"  s    
zFSSpecHandler.__eq__c                 C   s   t |tƒr| j|jkS tS r=   rA   rC   r   r   r   Ú__ne__'  s    
zFSSpecHandler.__ne__c                 C   s$   | j j}t|tƒr|d }d |¡S )Nr   z
fsspec+{0})r>   Úprotocolr*   Úlistr   )r?   rG   r   r   r   Úget_type_name,  s    
zFSSpecHandler.get_type_namec                 C   s   |S r=   r   ©r?   r$   r   r   r   r   2  s    zFSSpecHandler.normalize_pathc                 C   sR   |d }|d dkrt j}n|d dkr4t j}d }nt j}t| ||| dd ¡d�S )NÚsizer!   ÚfileÚ	directoryÚmtime)rK   rN   )r   ÚFiler"   ÚUnknownr   Úget)r$   ÚinforK   Zftyper   r   r   Ú_create_file_info5  s    zFSSpecHandler._create_file_infoc              	   C   sZ   g }|D ]L}z| j  |¡}W n$ ty@   | t|tjƒ¡ Y q0 | |  ||¡¡ q|S r=   )r>   rR   ÚFileNotFoundErrorÚappendr   r   r2   rS   )r?   ÚpathsÚinfosr$   rR   r   r   r   r    B  s    zFSSpecHandler.get_file_infoc                 C   sŽ   | j  |j¡s<| j  |j¡r(t|jƒ‚n|jr2g S t|jƒ‚|jrHd }nd}g }| j j|j|ddd�}| 	¡ D ]\}}| 
|  ||¡¡ qn|S )Né   T)ÚmaxdepthZwithdirsÚdetail)r>   ÚisdirÚbase_dirÚexistsÚNotADirectoryErrorZallow_not_foundrT   r9   ÚfindÚitemsrU   rS   )r?   ÚselectorrY   rW   Zselected_filesr$   rR   r   r   r   Úget_file_info_selectorM  s     

ÿz$FSSpecHandler.get_file_info_selectorc                 C   s,   z| j j||d� W n ty&   Y n0 d S )N)Zcreate_parents)r>   ÚmkdirÚFileExistsError)r?   r$   r9   r   r   r   Ú
create_dire  s    zFSSpecHandler.create_dirc                 C   s   | j j|dd� d S ©NTr8   )r>   ÚrmrJ   r   r   r   Ú
delete_dirl  s    zFSSpecHandler.delete_dirc                 C   sx   z| j j|dd�}W n ty2   |r,Y d S ‚ Y n0 |D ]:}| j  |¡rZ| j j|dd� q8| j  |¡r8| j  |¡ q8d S )NF)rZ   Tr8   )r>   ÚlistdirrT   r[   rg   Úisfile)r?   r$   Úmissing_dir_okZsubpathsÚsubpathr   r   r   Ú_delete_dir_contentso  s    z"FSSpecHandler._delete_dir_contentsc                 C   s*   |  d¡dkrtd|dƒ‚|  ||¡ d S )Nú/Ú z$delete_dir_contents called on path 'ú')Ústripr#   rm   )r?   r$   rk   r   r   r   Údelete_dir_contents|  s
    ÿz!FSSpecHandler.delete_dir_contentsc                 C   s   |   d¡ d S )Nrn   )rm   )r?   r   r   r   Údelete_root_dir_contents‚  s    z&FSSpecHandler.delete_root_dir_contentsc                 C   s$   | j  |¡st|ƒ‚| j  |¡ d S r=   )r>   r]   rT   rg   rJ   r   r   r   Údelete_file…  s    zFSSpecHandler.delete_filec                 C   s   | j j||dd� d S rf   )r>   Úmv©r?   ÚsrcÚdestr   r   r   ÚmoveŒ  s    zFSSpecHandler.movec                 C   s   | j  ||¡ d S r=   )r>   Úcopyrv   r   r   r   Ú	copy_file�  s    zFSSpecHandler.copy_filec                 C   s8   ddl m} | j |¡s t|ƒ‚|| jj|dd�dd�S ©Nr   ©Ú
PythonFileÚrb©ÚmodeÚr©Úpyarrowr~   r>   rj   rT   Úopen©r?   r$   r~   r   r   r   Úopen_input_stream–  s    zFSSpecHandler.open_input_streamc                 C   s8   ddl m} | j |¡s t|ƒ‚|| jj|dd�dd�S r|   rƒ   r†   r   r   r   Úopen_input_filež  s    zFSSpecHandler.open_input_filec                 C   s$   ddl m} || jj|dd�dd�S )Nr   r}   Úwbr€   Úw©r„   r~   r>   r…   ©r?   r$   Úmetadatar~   r   r   r   Úopen_output_stream¦  s    z FSSpecHandler.open_output_streamc                 C   s$   ddl m} || jj|dd�dd�S )Nr   r}   Úabr€   rŠ   r‹   rŒ   r   r   r   Úopen_append_stream«  s    z FSSpecHandler.open_append_streamN)r-   Ú
__module__Ú__qualname__Ú__doc__r@   rE   rF   rI   r   ÚstaticmethodrS   r    rb   re   rh   rm   rr   rs   rt   ry   r{   r‡   rˆ   rŽ   r�   r   r   r   r   r.     s,   
r.   )FF)NF)NN)$r“   Zpyarrow.utilr   r   Zpyarrow._fsr   r   r   r   r   r	   r
   r   r   r   r   Z	FileStatsr   Zpyarrow._hdfsr   r   rU   Zpyarrow._gcsfsr   Zpyarrow._s3fsr   r   r   r   r   r   r(   r1   r5   r<   r.   r   r   r   r   Ú<module>   s8   4  ÿ
* ÿ
> ÿþK