a
    5PÎj„8  ã                   @   s¾   d dl Z d dlZd dlZd dlZd dlZd dlmZ d dl	Z
d dlmZmZmZmZ e ddd¡ZG dd„ dƒZG d	d
„ d
eƒZG dd„ deƒZG dd„ deƒZdd„ Zdd„ Zddd„ZdS )é    N)Újoin)Ú
implementsÚ_stringify_pathÚ_is_path_likeÚ	_DEPR_MSGzfilesystem.LocalFileSystemz2.0.0zfs.LocalFileSystemc                   @   s¦   e Zd ZdZdd„ Zdd„ Zd(dd„Zd	d
„ Zdd„ Zdd„ Z	d)dd„Z
dd„ Zdd„ Zd*dd„Zdd„ Zdd„ Zdd„ Zdd„ Zd+d!d"„Zd,d$d%„Zed&d'„ ƒZd S )-Ú
FileSystemz(
    Abstract filesystem interface.
    c                 C   s8   |   |d¡�}| ¡ W  d  ƒ S 1 s*0    Y  dS )zÜ
        Return contents of file as a bytes object.

        Parameters
        ----------
        path : str
            File path to read content from.

        Returns
        -------
        contents : bytes
        ÚrbN)ÚopenÚread)ÚselfÚpathÚf© r   úb/mounts/lovelace/software/anaconda3/envs/metadmg/lib/python3.9/site-packages/pyarrow/filesystem.pyÚcat)   s    zFileSystem.catc                 C   s   t ‚dS )z“
        Return list of file paths.

        Parameters
        ----------
        path : str
            Directory to list contents from.
        N©ÚNotImplementedError©r   r   r   r   r   Úls9   s    	zFileSystem.lsFc                 C   s   t ‚dS )zõ
        Delete the indicated file or directory.

        Parameters
        ----------
        path : str
            Path to delete.
        recursive : bool, default False
            If True, also delete child paths for directories.
        Nr   ©r   r   Ú	recursiver   r   r   ÚdeleteD   s    zFileSystem.deletec           	      C   sl   t |ƒ}|  |¡}|d dkr&|d S d}|  |¡D ]2\}}}|D ]"}|  ||¡}||  |¡d 7 }qBq4|S )zó
        Compute bytes used by all contents under indicated path in file tree.

        Parameters
        ----------
        path : str
            Can be a file path or directory.

        Returns
        -------
        usage : int
        ÚkindÚfileÚsizer   )r   ÚstatÚwalkÚ
_path_join)	r   r   Z	path_infoÚtotalÚrootÚdirectoriesÚfilesZ
child_pathÚabspathr   r   r   Ú
disk_usageQ   s    
zFileSystem.disk_usagec                 G   s   | j  |¡S ©N)Úpathsepr   )r   Úargsr   r   r   r   k   s    zFileSystem._path_joinc                 C   s   t dƒ‚dS )zl
        Information about a filesystem entry.

        Returns
        -------
        stat : dict
        úFileSystem.statNr   r   r   r   r   r   n   s    r'   c                 C   s   | j ||d�S )z.
        Alias for FileSystem.delete.
        ©r   )r   r   r   r   r   Úrmx   s    zFileSystem.rmc                 C   s   |   ||¡S )z.
        Alias for FileSystem.rename.
        )Úrename©r   r   Únew_pathr   r   r   Úmv~   s    zFileSystem.mvc                 C   s   t dƒ‚dS )z½
        Rename file, like UNIX mv command.

        Parameters
        ----------
        path : str
            Path to alter.
        new_path : str
            Path to move to.
        úFileSystem.renameNr   r+   r   r   r   r*   „   s    r.   Tc                 C   s   t ‚dS )zö
        Create a directory.

        Parameters
        ----------
        path : str
            Path to the directory.
        create_parents : bool, default True
            If the parent directories don't exists create them as well.
        Nr   ©r   r   Zcreate_parentsr   r   r   Úmkdir‘   s    zFileSystem.mkdirc                 C   s   t ‚dS )z‚
        Return True if path exists.

        Parameters
        ----------
        path : str
            Path to check.
        Nr   r   r   r   r   Úexistsž   s    	zFileSystem.existsc                 C   s   t ‚dS )zŠ
        Return True if path is a directory.

        Parameters
        ----------
        path : str
            Path to check.
        Nr   r   r   r   r   Úisdir©   s    	zFileSystem.isdirc                 C   s   t ‚dS )z…
        Return True if path is a file.

        Parameters
        ----------
        path : str
            Path to check.
        Nr   r   r   r   r   Úisfile´   s    	zFileSystem.isfilec                 C   s   t ‚dS )zf
        Returns True if this FileSystem is a unix-style file store with
        directories.
        Nr   ©r   r   r   r   Ú_isfilestore¿   s    zFileSystem._isfilestoreNc           	      C   s,   ddl m} ||||| d�}|j|||d�S )a]  
        Read Parquet data from path in file system. Can read from a single file
        or a directory of files.

        Parameters
        ----------
        path : str
            Single file path or directory
        columns : List[str], optional
            Subset of columns to read.
        metadata : pyarrow.parquet.FileMetaData
            Known metadata to validate files against.
        schema : pyarrow.parquet.Schema
            Known schema to validate files against. Alternative to metadata
            argument.
        use_threads : bool, default True
            Perform multi-threaded column reads.
        use_pandas_metadata : bool, default False
            If True and file has custom pandas schema metadata, ensure that
            index columns are also loaded.

        Returns
        -------
        table : pyarrow.Table
        r   )ÚParquetDataset)ÚschemaÚmetadataÚ
filesystem)ÚcolumnsÚuse_threadsÚuse_pandas_metadata)Zpyarrow.parquetr6   r
   )	r   r   r:   r8   r7   r;   r<   r6   Zdatasetr   r   r   Úread_parquetÆ   s    ÿÿzFileSystem.read_parquetr   c                 C   s   t ‚dS )ú3
        Open file for reading or writing.
        Nr   ©r   r   Úmoder   r   r   r	   ç   s    zFileSystem.openc                 C   s   dS )Nú/r   r4   r   r   r   r%   í   s    zFileSystem.pathsep)F)F)T)NNNTF)r   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   r   r#   r   r   r)   r-   r*   r0   r1   r2   r3   r5   r=   r	   Úpropertyr%   r   r   r   r   r   $   s*   



  ÿ
!
r   c                       sÎ   e Zd ZdZ‡ fdd„Zedd„ ƒZedd„ ƒZee	j
ƒdd	„ ƒZ
ee	jƒddd„ƒZee	jƒdd„ ƒZee	jƒdd„ ƒZee	jƒdd„ ƒZee	jƒdd„ ƒZee	jƒddd„ƒZedd„ ƒZdd„ Z‡  ZS )ÚLocalFileSystemNc                    s   t jttdd� tƒ  ¡  d S ©Né   ©Ú
stacklevel)ÚwarningsÚwarnÚ_FS_DEPR_MSGÚFutureWarningÚsuperÚ__init__r4   ©Ú	__class__r   r   rQ   ö   s    zLocalFileSystem.__init__c                 C   sJ   | j d u rDt ¡ �" t d¡ tƒ | _ W d   ƒ n1 s:0    Y  | j S )NÚignore)Ú	_instancerL   Úcatch_warningsÚsimplefilterrG   ©Úclsr   r   r   Ú_get_instanceú   s
    


&zLocalFileSystem._get_instancec                 C   s   t jttdd� |  ¡ S rH   )rL   rM   rN   rO   rZ   rX   r   r   r   Úget_instance  s    zLocalFileSystem.get_instancec                    s$   t ˆ ƒ‰ t‡ fdd„t ˆ ¡D ƒƒS )Nc                 3   s   | ]}t ˆ |ƒV  qd S r$   )Úpjoin©Ú.0Úx©r   r   r   Ú	<genexpr>
  ó    z%LocalFileSystem.ls.<locals>.<genexpr>)r   ÚsortedÚosÚlistdirr   r   r`   r   r     s    zLocalFileSystem.lsTc                 C   s&   t |ƒ}|rt |¡ n
t |¡ d S r$   )r   rd   Úmakedirsr0   r/   r   r   r   r0     s    zLocalFileSystem.mkdirc                 C   s   t |ƒ}tj |¡S r$   )r   rd   r   r2   r   r   r   r   r2     s    zLocalFileSystem.isdirc                 C   s   t |ƒ}tj |¡S r$   )r   rd   r   r3   r   r   r   r   r3     s    zLocalFileSystem.isfilec                 C   s   dS )NTr   r4   r   r   r   r5     s    zLocalFileSystem._isfilestorec                 C   s   t |ƒ}tj |¡S r$   )r   rd   r   r1   r   r   r   r   r1   "  s    zLocalFileSystem.existsr   c                 C   s   t |ƒ}t||d�S ©r>   )r@   )r   r	   r?   r   r   r   r	   '  s    zLocalFileSystem.openc                 C   s   t jjS r$   )rd   r   Úsepr4   r   r   r   r%   /  s    zLocalFileSystem.pathsepc                 C   s   t |ƒ}t |¡S )z8
        Directory tree generator, see os.walk.
        )r   rd   r   r   r   r   r   r   3  s    zLocalFileSystem.walk)T)r   )rB   rC   rD   rU   rQ   ÚclassmethodrZ   r[   r   r   r   r0   r2   r3   r5   r1   r	   rF   r%   r   Ú__classcell__r   r   rR   r   rG   ò   s.   







rG   c                   @   s®   e Zd ZdZdd„ Zeejƒdd„ ƒZeejƒdd„ ƒZeej	ƒdd	„ ƒZ	eej
ƒddd„ƒZ
eejƒdd„ ƒZeejƒddd„ƒZeejƒddd„ƒZddd„Zdd„ ZdS )ÚDaskFileSystemzJ
    Wraps s3fs Dask filesystem implementation like s3fs, gcsfs, etc.
    c                 C   s   t jdtdd� || _d S )Nz~The pyarrow.filesystem.DaskFileSystem/S3FSWrapper are deprecated as of pyarrow 3.0.0, and will be removed in a future version.rI   rJ   )rL   rM   rO   Úfs)r   rl   r   r   r   rQ   @  s
    ýzDaskFileSystem.__init__c                 C   s   t dƒ‚d S ©NzUnsupported file system APIr   r   r   r   r   r2   G  s    zDaskFileSystem.isdirc                 C   s   t dƒ‚d S rm   r   r   r   r   r   r3   K  s    zDaskFileSystem.isfilec                 C   s   dS )zh
        Object Stores like S3 and GCSFS are based on key lookups, not true
        file-paths.
        Fr   r4   r   r   r   r5   O  s    zDaskFileSystem._isfilestoreFc                 C   s   t |ƒ}| jj||d�S )Nr(   )r   rl   r)   r   r   r   r   r   W  s    zDaskFileSystem.deletec                 C   s   t |ƒ}| j |¡S r$   )r   rl   r1   r   r   r   r   r1   \  s    zDaskFileSystem.existsTc                 C   s(   t |ƒ}|r| j |¡S | j |¡S d S r$   )r   rl   Zmkdirsr0   r/   r   r   r   r0   a  s    zDaskFileSystem.mkdirr   c                 C   s   t |ƒ}| jj||d�S rg   )r   rl   r	   r?   r   r   r   r	   i  s    zDaskFileSystem.openc                 C   s   t |ƒ}| jj||d�S )N)Údetail)r   rl   r   )r   r   rn   r   r   r   r   q  s    zDaskFileSystem.lsc                 C   s   t |ƒ}| j |¡S )z9
        Directory tree generator, like os.walk.
        )r   rl   r   r   r   r   r   r   u  s    zDaskFileSystem.walkN)F)T)r   )F)rB   rC   rD   rE   rQ   r   r   r2   r3   r5   r   r1   r0   r	   r   r   r   r   r   r   rk   ;  s$   




rk   c                   @   s:   e Zd Zeejƒdd„ ƒZeejƒdd„ ƒZd	dd„ZdS )
ÚS3FSWrapperc                 C   sZ   t t|ƒƒ}z4| j |¡}t|ƒdkr8|d |kr8W dS W dS W n tyT   Y dS 0 d S )Né   r   FT©Ú_sanitize_s3r   rl   r   ÚlenÚOSError©r   r   Úcontentsr   r   r   r2     s    
zS3FSWrapper.isdirc                 C   sL   t t|ƒƒ}z&| j |¡}t|ƒdko0|d |kW S  tyF   Y dS 0 d S )Nrp   r   Frq   ru   r   r   r   r3   ‹  s    zS3FSWrapper.isfileFc                 #   sÀ   t t|ƒƒ}tƒ ‰ tƒ }t| jj||d�ƒD ]<}|d }|d dkrPˆ  |¡ q,|d dkr^q,| |¡ q,t‡ fdd„|D ƒƒ}tdd„ ˆ D ƒƒ‰ |ˆ |fV  ˆ D ]}| j||d�E d	H  q¢d	S )
z˜
        Directory tree generator, like os.walk.

        Generator version of what is in s3fs, which yields a flattened list of
        files.
        )ÚrefreshÚKeyZStorageClassZ	DIRECTORYZBUCKETc                    s"   g | ]}|ˆ vrt  |¡d  ‘qS ©rp   ©Ú	posixpathÚsplit)r^   r   ©r    r   r   Ú
<listcomp>©  s   ÿz$S3FSWrapper.walk.<locals>.<listcomp>c                 S   s   g | ]}t  |¡d  ‘qS ry   rz   r]   r   r   r   r~   «  s   ÿN)	rr   r   ÚsetÚlistrl   Z_lsÚaddrc   r   )r   r   rw   r!   ÚkeyÚ	directoryr   r}   r   r   ”  s"    ÿzS3FSWrapper.walkN)F)rB   rC   rD   r   r   r2   r3   r   r   r   r   r   ro   }  s
   

ro   c                 C   s   |   d¡r|  dd¡S | S d S )Nzs3://Ú )Ú
startswithÚreplacer`   r   r   r   rr   ´  s    
rr   c                 C   sN   t | ƒ}t|tƒsFdtjv r6tjd }t| |jƒr6| S td |¡ƒ‚n| S d S )NÚfsspeczUnrecognized filesystem: {})	ÚtypeÚ
issubclassr   ÚsysÚmodulesÚ
isinstanceZAbstractFileSystemrt   Úformat)rl   Zfs_typer‡   r   r   r   Ú_ensure_filesystem»  s    


rŽ   c           	      C   s,  t | ƒs |durtdƒ‚|| fS |durdt|ƒ}t|tƒrDt| ƒ}nt| tƒsXtdƒ‚n| }||fS t| ƒ}tj	 
|¡}|jdksŒ|jdkrü|j d¡}|d }|dkr®d	}n|jd
 | }d}t|ƒdkrä|d  ¡ rät|d ƒ}tjj||d�}|j}n(|jdk�rt ¡ }|j}nt ¡ }|}||fS )zn
    Return filesystem from path which could be an HDFS URI, a local URI,
    or a plain filesystem path.
    NzVfilesystem passed but where is file-like, so there is nothing to open with filesystem.zPExpected string path; path-like objects are only allowed with a local filesystemÚhdfsZviewfsú:r   r„   Údefaultz://rI   rp   )ÚhostÚportr   )r   Ú
ValueErrorrŽ   rŒ   rG   r   ÚstrÚ	TypeErrorÚurllibÚparseÚurlparseÚschemeÚnetlocr|   rs   Ú	isnumericÚintÚpar�   Ú_connectr   rZ   )	Úwherer9   r   Z
parsed_uriZnetloc_splitr’   r“   rl   Zfs_pathr   r   r   Úresolve_filesystem_and_pathÎ  sB    


ÿr¡   )N)rd   r{   rŠ   Úurllib.parser—   rL   Úos.pathr   r\   Zpyarrowrž   Zpyarrow.utilr   r   r   r   r�   rN   r   rG   rk   ro   rr   rŽ   r¡   r   r   r   r   Ú<module>   s$   ÿ OIB7