a
    5PÎj
'  ã                   @   s¤   d dl Z d dlmZ d dlmZmZmZmZ d dlmZ	 d dl
mZ d dlmZ d dlmZ dd„ ZG d	d
„ d
ƒZdd„ Zh d£Zddd„Zddd„Zddd„ZdS )é    N)Ú_pandas_api)ÚCodecÚTableÚconcat_tablesÚschema)Ú_feather)ÚFeatherError)ÚVersionc                   C   s   t jtdƒk rtdƒ‚d S )Nz0.17.0z!feather requires pandas >= 0.17.0)r   Zloose_versionr	   ÚImportError© r   r   ú_/mounts/lovelace/software/anaconda3/envs/metadmg/lib/python3.9/site-packages/pyarrow/feather.pyÚ_check_pandas_version   s    r   c                   @   s6   e Zd ZdZddd„Zddd„Zdd	„ Zdd
d„ZdS )ÚFeatherDataseta  
    Encapsulates details of reading a list of Feather files.

    Parameters
    ----------
    path_or_paths : List[str]
        A list of file names
    validate_schema : bool, default True
        Check that individual file schemas are all the same / compatible
    Tc                 C   s   || _ || _d S ©N)ÚpathsÚvalidate_schema)ÚselfZpath_or_pathsr   r   r   r   Ú__init__/   s    zFeatherDataset.__init__Nc                 C   sj   t | jd |d�}|g| _|j| _| jdd… D ].}t ||d�}| jrR|  ||¡ | j |¡ q0t| jƒS )a,  
        Read multiple feather files as a single pyarrow.Table

        Parameters
        ----------
        columns : List[str]
            Names of columns to read from the file

        Returns
        -------
        pyarrow.Table
            Content of the file as a table (of columns)
        r   ©Úcolumnsé   N)Ú
read_tabler   Z_tablesr   r   Úvalidate_schemasÚappendr   )r   r   Z_filÚpathÚtabler   r   r   r   3   s    zFeatherDataset.read_tablec                 C   s(   | j  |j ¡s$td || j |j ¡ƒ‚d S )Nz-Schema in {!s} was different. 
{!s}

vs

{!s})r   ÚequalsÚ
ValueErrorÚformat)r   Zpiecer   r   r   r   r   L   s
    ýzFeatherDataset.validate_schemasc                 C   s   t ƒ  | j|d�j|d�S )a¡  
        Read multiple Parquet files as a single pandas DataFrame

        Parameters
        ----------
        columns : List[str]
            Names of columns to read from the file
        use_threads : bool, default True
            Use multiple threads when converting to pandas

        Returns
        -------
        pandas.DataFrame
            Content of the file as a pandas DataFrame (of columns)
        r   ©Úuse_threads©r   r   Z	to_pandas)r   r   r    r   r   r   Úread_pandasS   s    ÿzFeatherDataset.read_pandas)T)N)NT)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   r   r"   r   r   r   r   r   #   s
   

r   c                 C   sN   |j dkrd S |jt ¡ t ¡ fv r4td | ¡ƒ‚ntd | t|jƒ¡ƒ‚d S )Nr   zqColumn '{}' exceeds 2GB maximum capacity of a Feather binary column. This restriction may be lifted in the futurezkColumn '{}' of type {} was chunked on conversion to Arrow and cannot be currently written to Feather format)Z
num_chunksÚtypeÚextÚbinaryÚstringr   r   Ústr)ÚnameÚcolr   r   r   Úcheck_chunked_overflowh   s    
þ
þr.   >   Úlz4ZzstdZuncompressedé   c                 C   sˆ  t jr(tƒ  t jr(t| t jjƒr(|  ¡ } t  | ¡r–|dkr@d}n|dkrNd}nt	dƒ‚t
j| |d�}|dkršt|jjƒD ]\}}	|| }
t|	|
ƒ qxn| }|dkrät|jƒtt|jƒƒkrÂt	dƒ‚|durÒt	dƒ‚|durât	d	ƒ‚n<|du rüt d
¡rüd}n$|du�r |tv�r t	d |t¡ƒ‚ztj||||||d� W nH t�y‚   t|tƒ�r|zt |¡ W n tj�yz   Y n0 ‚ Y n0 dS )aš  
    Write a pandas.DataFrame to Feather format.

    Parameters
    ----------
    df : pandas.DataFrame or pyarrow.Table
        Data to write out as Feather format.
    dest : str
        Local destination path.
    compression : string, default None
        Can be one of {"zstd", "lz4", "uncompressed"}. The default of None uses
        LZ4 for V2 files if it is available, otherwise uncompressed.
    compression_level : int, default None
        Use a compression level particular to the chosen compressor. If None
        use the default compression level
    chunksize : int, default None
        For V2 files, the internal maximum size of Arrow RecordBatch chunks
        when writing the Arrow IPC file format. None means use the default,
        which is currently 64K
    version : int, default 2
        Feather file version. Version 2 is the current. Version 1 is the more
        limited legacy format
    r   Fr0   Nz%Version value should either be 1 or 2)Úpreserve_indexz'cannot serialize duplicate column namesz2Feather V1 files do not support compression optionz0Feather V1 files do not support chunksize optionZ	lz4_framer/   z1compression="{}" not supported, must be one of {})ÚcompressionÚcompression_levelÚ	chunksizeÚversion)r   Zhave_pandasr   Z
has_sparseÚ
isinstanceÚpdZSparseDataFrameZto_denseZis_data_framer   r   Zfrom_pandasÚ	enumerater   Únamesr.   ÚlenZcolumn_namesÚsetr   Zis_availableÚ_FEATHER_SUPPORTED_CODECSr   r   Úwrite_featherÚ	Exceptionr+   ÚosÚremoveÚerror)ZdfÚdestr2   r3   r4   r5   r1   r   Úir,   r-   r   r   r   r=   z   s\    ÿ


ÿþ
þ
r=   TFc                 C   s   t ƒ  t| |||d�j|d�S )a  
    Read a pandas.DataFrame from Feather format. To read as pyarrow.Table use
    feather.read_table.

    Parameters
    ----------
    source : str file path, or file-like object
        You can use MemoryMappedFile as source, for explicitly use memory map.
    columns : sequence, optional
        Only read a specific set of columns. If not provided, all columns are
        read.
    use_threads : bool, default True
        Whether to parallelize reading using multiple threads. If false the
        restriction is used in the conversion to Pandas as well as in the
        reading from Feather format.
    memory_map : boolean, default False
        Use memory mapping when opening file on disk, when source is a str.

    Returns
    -------
    df : pandas.DataFrame
    )r   Ú
memory_mapr    r   r!   )Úsourcer   r    rD   r   r   r   Úread_featherÏ   s    þþrF   c                 C   s¸   t j| ||d�}|du r | ¡ S dd„ |D ƒ}ttdd„ |ƒƒrL| |¡}n<ttdd„ |ƒƒrj| |¡}ndd„ |D ƒ}td	 ||¡ƒ‚|j	d
k r–|S t
t|ƒƒ|krª|S | |¡S dS )aW  
    Read a pyarrow.Table from Feather format

    Parameters
    ----------
    source : str file path, or file-like object
        You can use MemoryMappedFile as source, for explicitly use memory map.
    columns : sequence, optional
        Only read a specific set of columns. If not provided, all columns are
        read.
    memory_map : boolean, default False
        Use memory mapping when opening file on disk, when source is a str
    use_threads : bool, default True
        Whether to parallelize reading using multiple threads.

    Returns
    -------
    table : pyarrow.Table
    )Zuse_memory_mapr    Nc                 S   s   g | ]}t |ƒ‘qS r   )r'   )Ú.0Úcolumnr   r   r   Ú
<listcomp>  ó    zread_table.<locals>.<listcomp>c                 S   s   | t kS r   )Úint©Útr   r   r   Ú<lambda>  rJ   zread_table.<locals>.<lambda>c                 S   s   | t kS r   )r+   rL   r   r   r   rN   	  rJ   c                 S   s   g | ]
}|j ‘qS r   )r#   )rG   rM   r   r   r   rI     rJ   z<Columns must be indices or names. Got columns {} of types {}é   )r   ZFeatherReaderÚreadÚallÚmapZread_indicesZ
read_namesÚ	TypeErrorr   r5   Úsortedr;   Úselect)rE   r   rD   r    ÚreaderZcolumn_typesr   Zcolumn_type_namesr   r   r   r   ì   s&    ÿþ
r   )NNNr0   )NTF)NFT)r?   Zpyarrow.pandas_compatr   Zpyarrow.libr   r   r   r   Úlibr(   Zpyarrowr   Zpyarrow._featherr   Zpyarrow.vendored.versionr	   r   r   r.   r<   r=   rF   r   r   r   r   r   Ú<module>   s   E  ÿ
U
