a
    ,ghG                     @   s0  d dl Z d dlZd dlZd dlZd dlmZmZm	Z	 zd dl
Z
W n eyV   e	Z
Y n0 zd dlmZ W n ey   dd ZY n0 dd Zdd	 Zd
d Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zd d! Zd"d# Zd$d% Zd&d' Zd(d) Zd*d+ Z d,d- Z!d.d/ Z"dS )0    N)SerializationContext	py_bufferbuiltin_pickle)descr_to_dtypec                 C   s  t | trt| S t | tr<t| d }t|| d fS g }d}| D ]z}t|dkrj|\}}t|}n|\}}}tt||f}|dko|jtju o|j	du }|s|
|||f ||j7 }qHt| \}	}
}dd |	D }t| \}}	t|	|
|||dS )	ad  
        descr may be stored as dtype.descr, which is a list of (name, format,
        [shape]) tuples where format may be a str or a tuple.  Offsets are not
        explicitly saved, rather empty fields with name, format == '', '|Vn'
        are added as padding.  This function reverses the process, eliminating
        the empty padding fields.
        r          Nc                 s   s$   | ]}t |tr|nd |fV  qd S N)
isinstancetuple).0n r   e/mounts/lovelace/software/anaconda3/envs/metaDMG/lib/python3.9/site-packages/pyarrow/serialization.py	<genexpr>I       z!descr_to_dtype.<locals>.<genexpr>)namesformatstitlesoffsetsitemsize)r
   strnpdtyper   r   lentypevoidr   appendr   zip)descrdtfieldsoffsetfieldnameZ	descr_strshapeZis_padr   r   r   Znametupsr   r   r   r   r   %   s.    





r   c                 C   s   d | }tj|tdd d S )Nz'pyarrow.{}' is deprecated as of 2.0.0 and will be removed in a future version. Use pickle or the pyarrow IPC functionality instead.   )
stacklevel)formatwarningswarnFutureWarning)r$   msgr   r   r   _deprecate_serializationO   s
    r-   c                 C   sT   | j jdkr8| jjst| } | dtjj	| j fS | 
 tjj	| j fS d S N|Ouint8)r   r   flagsc_contiguousr   ascontiguousarrayviewlibr(   dtype_to_descrtolistobjr   r   r   _serialize_numpy_array_list\   s
    
r:   c                 C   sV   | d dkr6| d j tjks J | d t| d S tj| d t | d dS d S )Nr   r/   r   )r   )r   r   r0   r4   r   arraydatar   r   r   _deserialize_numpy_array_listg   s    r>   c                 C   sZ   | j jdkr<| jjs t| j} | jdtjj	
| j fS | j tjj	
| j fS d S r.   )r   r   r1   r2   r   r3   Ar4   r5   r(   r6   r7   r8   r   r   r   _serialize_numpy_matrixo   s
    r@   c                 C   sb   | d dkr@| d j tjks J tj| d t| d ddS tj| d t | d ddS d S )Nr   r/   r   F)copy)r   rA   )r   r   r0   matrixr4   r   r<   r   r   r   _deserialize_numpy_matrixz   s    rC   c                 C   sJ   t  }t j|| jd}||  W d    n1 s80    Y  | S N)schema)paBufferOutputStreamRecordBatchStreamWriterrE   Zwrite_batchgetvalue)batchoutput_streamwrr   r   r   _serialize_pyarrow_recordbatch   s    (rM   c                 C   s6   t | }| W  d    S 1 s(0    Y  d S r	   )rF   RecordBatchStreamReaderZread_next_batchbufreaderr   r   r    _deserialize_pyarrow_recordbatch   s    rR   c                 C   s   t j| gdg}t|S )Nr   )rF   RecordBatchZfrom_arraysrM   )r;   rJ   r   r   r   _serialize_pyarrow_array   s    rT   c                 C   s   t | }|jd S Nr   )rR   columns)rP   rJ   r   r   r   _deserialize_pyarrow_array   s    rW   c                 C   sJ   t  }t j|| jd}||  W d    n1 s80    Y  | S rD   )rF   rG   rH   rE   Zwrite_tablerI   )tablerK   rL   r   r   r   _serialize_pyarrow_table   s    (rY   c                 C   s6   t | }| W  d    S 1 s(0    Y  d S r	   )rF   rN   Zread_allrO   r   r   r   _deserialize_pyarrow_table   s    rZ   c                 C   s   t j| t jd}t|S )N)protocol)r   dumpsHIGHEST_PROTOCOLr   )xZpickledr   r   r   _pickle_to_buffer   s    r_   c                 C   s   t | }t|S r	   )
memoryviewr   loads)r=   Zas_memoryviewr   r   r   _load_pickle_from_buffer   s    rb   c                    s4  zdd l W n ty    Y d S 0 dd lm dfddfdd fdd} fd	d
}| jjd||d | jjdttd t	j
drt	j
jdr| jj
jjjdttd t	j
jdr| jj
jjjdttd t	j
jdr| jj
jjjdttd | jjd d d S )Nr   z{0} serialization is not supported.
Note that {0} is planned to be deprecated in pandas future releases.
See https://github.com/pandas-dev/pandas/issues/19239 for more information.c                    s,   j jr"t|  jr"td| S )NSparseDataFrame)_pandas_api
has_sparser
   rc   NotImplementedErrorr(   Zdataframe_to_serialized_dictr8   )pdpdcompatsparse_type_error_msgr   r   _serialize_pandas_dataframe   s    
zE_register_custom_pandas_handlers.<locals>._serialize_pandas_dataframec                    s
     | S r	   )Zserialized_dict_to_dataframer<   )rh   r   r   _deserialize_pandas_dataframe   s    zG_register_custom_pandas_handlers.<locals>._deserialize_pandas_dataframec                    s6   j jr"t| jr"td | j| iS )NSparseSeries)rd   re   r
   rl   rf   r(   	DataFramer$   r8   )rj   rg   rh   ri   r   r   _serialize_pandas_series   s    
zB_register_custom_pandas_handlers.<locals>._serialize_pandas_seriesc                    s    | }||j d  S rU   )rV   )r=   Zdeserialized)rk   r   r   _deserialize_pandas_series   s    zD_register_custom_pandas_handlers.<locals>._deserialize_pandas_seriesz	pd.SeriesZcustom_serializerZcustom_deserializerzpd.Indexarraysintervalz%pd.core.arrays.interval.IntervalArrayperiodz!pd.core.arrays.period.PeriodArray	datetimesz&pd.core.arrays.datetimes.DatetimeArrayzpd.DataFrame)ZpandasImportErrorZpyarrow.pandas_compatZpandas_compatregister_typeZSeriesIndexr_   rb   hasattrcorerq   rr   ZIntervalArrayrs   ZPeriodArrayrt   ZDatetimeArrayrm   )contextrn   ro   r   )rk   rj   rg   rh   ri   r    _register_custom_pandas_handlers   s`    		


r{   c              	      s   t d zfdd l dd } fdd} j j j j j j j j	 j
f	D ]}| j|d|j ||d qNW n ty   Y n0 d S )	N%register_torch_serialization_handlersr   c                 S   sH   | j r8tjj|    |    jt	| j
dS |   S d S )N)r%   )Z	is_sparserF   SparseCOOTensor
from_numpyZ_valuesdetachnumpyZ_indicesTlistr%   r8   r   r   r   _serialize_torch_tensor  s    zFregister_torch_serialization_handlers.<locals>._serialize_torch_tensorc                    sJ   t | tjr< j|  d j|  d d d df | jdS  | S d S )Nr   r   )indicesvaluessize)r
   rF   r}   Zsparse_coo_tensorto_numpyr   r%   r~   r<   torchr   r   _deserialize_torch_tensor  s    zHregister_torch_serialization_handlers.<locals>._deserialize_torch_tensorztorch.rp   )r-   r   ZFloatTensorZDoubleTensorZ
HalfTensorZ
ByteTensorZ
CharTensorZShortTensorZ	IntTensorZ
LongTensorZTensorrv   __name__ru   )serialization_contextr   r   tr   r   r   r|     s     		
r|   c           	      C   s   dd }dd }| j tjd||d dd }d	d
 }| j tjd||d dd }dd }| j tjd||d dd }dd }| j tjd||d d S )Nc                 S   s   t | S r	   )r   r8   r   r   r   _serialize_deque5  s    zF_register_collections_serialization_handlers.<locals>._serialize_dequec                 S   s
   t | S r	   )collectionsdequer<   r   r   r   _deserialize_deque8  s    zH_register_collections_serialization_handlers.<locals>._deserialize_dequezcollections.dequerp   c                 S   s   t |  t |  fS r	   r   keysr   r8   r   r   r   _serialize_ordered_dict@  s    zM_register_collections_serialization_handlers.<locals>._serialize_ordered_dictc                 S   s   t t| d | d S Nr   r   )r   OrderedDictr   r<   r   r   r   _deserialize_ordered_dictC  s    zO_register_collections_serialization_handlers.<locals>._deserialize_ordered_dictzcollections.OrderedDictc                 S   s   t |  t |  | jfS r	   )r   r   r   default_factoryr8   r   r   r   _serialize_default_dictK  s    zM_register_collections_serialization_handlers.<locals>._serialize_default_dictc                 S   s   t | d t| d | d S )Nr   r   r   )r   defaultdictr   r<   r   r   r   _deserialize_default_dictN  s    zO_register_collections_serialization_handlers.<locals>._deserialize_default_dictzcollections.defaultdictc                 S   s   t |  t |  fS r	   r   r8   r   r   r   _serialize_counterV  s    zH_register_collections_serialization_handlers.<locals>._serialize_counterc                 S   s   t tt| d | d S r   )r   Counterdictr   r<   r   r   r   _deserialize_counterY  s    zJ_register_collections_serialization_handlers.<locals>._deserialize_counterzcollections.Counter)rv   r   r   r   r   r   )	r   r   r   r   r   r   r   r   r   r   r   r   ,_register_collections_serialization_handlers4  s8    r   c                    s   zxddl m}m}m}mmmm   fdd}dd }| j|d||d | j|d	||d | j|d
||d W n t	y   Y n0 d S )Nr   )
csr_matrix
csc_matrix
coo_matrixisspmatrix_cooisspmatrix_csrisspmatrix_csc
isspmatrixc                    sz   | rdt j| fS | r0dt j| fS | rHdt j| fS  | rddt j|  fS td| d d S )Ncoocsrcsc%Serialization of {} is not supported.r   )rF   r}   Z
from_scipyZSparseCSRMatrixZSparseCSCMatrixZto_coorf   r(   r8   r   r   r   r   r   r   _serialize_scipy_sparsel  s    z9_register_scipy_handlers.<locals>._serialize_scipy_sparsec                 S   sX   | d dkr| d   S | d dkr0| d   S | d dkrH| d   S | d   S d S )Nr   r   r   r   r   )Zto_scipyr<   r   r   r   _deserialize_scipy_sparse}  s    z;_register_scipy_handlers.<locals>._deserialize_scipy_sparsezscipy.sparse.coo.coo_matrixrp   zscipy.sparse.csr.csr_matrixzscipy.sparse.csc.csc_matrix)
Zscipy.sparser   r   r   r   r   r   r   rv   ru   )r   r   r   r   r   r   r   r   r   _register_scipy_handlersf  s*    $
r   c                    sP   z8dd l   fdd} fdd}| j jd||d W n tyJ   Y n0 d S )Nr   c                    s0   t |  jrdtj| fS td jd S )Nr   r   )r
   COOrF   r}   Zfrom_pydata_sparserf   r(   r8   sparser   r   _serialize_pydata_sparse  s
    
zB_register_pydata_sparse_handlers.<locals>._serialize_pydata_sparsec                    sD   | d dkr@| d   \}} j|d d df |j| d jdS d S )Nr   r   r   )r=   coordsr%   )r   r   r   r%   )r=   Z
data_arrayr   r   r   r   _deserialize_pydata_sparse  s    zD_register_pydata_sparse_handlers.<locals>._deserialize_pydata_sparsez
sparse.COOrp   )r   rv   r   ru   )r   r   r   r   r   r    _register_pydata_sparse_handlers  s    
r   c                 C   s   | j tddd dd d | j tdd ddd	 | j td
dd	 | j tjdttd | j tjdtt	d | j t
jdttd | j t
jdttd | j t
jdttd t|  t|  t|  t|  d S )Nintc                 S   s   t | S r	   )r   r8   r   r   r   <lambda>  r   z:_register_default_serialization_handlers.<locals>.<lambda>c                 S   s   t | S r	   )r   r<   r   r   r   r     r   rp   c                   S   s   dS rU   r   r   r   r   r   r     r   functionT)pickler   z	np.matrixznp.arrayzpyarrow.Arrayzpyarrow.RecordBatchzpyarrow.Table)rv   r   r   r   rB   r@   rC   Zndarrayr:   r>   rF   ZArrayrT   rW   rS   rM   rR   TablerY   rZ   r   r{   r   r   r   r   r   r   (_register_default_serialization_handlers  sN    r   c                 C   s   t d t|  d S )N'register_default_serialization_handlers)r-   r   r   r   r   r   r     s    r   c                  C   s   t d t } t|  | S )Ndefault_serialization_context)r-   r   r   )rz   r   r   r   r     s    r   )#r   r)   r   r   ZpyarrowrF   Zpyarrow.libr   r   r   Zcloudpickleru   Znumpy.lib.formatr   r-   r:   r>   r@   rC   rM   rR   rT   rW   rY   rZ   r_   rb   r{   r|   r   r   r   r   r   r   r   r   r   r   <module>   sB   
*	Q&2;2