a
    5PÎjG  ã                   @   s0  d dl Z d dlZd dlZd dlZd dlmZmZm	Z	 zd dl
Z
W n eyV   e	Z
Y n0 zd dlmZ W n ey‚   dd„ ZY n0 dd„ Zdd	„ Zd
d„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zd d!„ Zd"d#„ Zd$d%„ Zd&d'„ Zd(d)„ Zd*d+„ Z d,d-„ Z!d.d/„ Z"dS )0é    N)ÚSerializationContextÚ	py_bufferÚbuiltin_pickle)Údescr_to_dtypec                 C   s  t | tƒrt | ¡S t | tƒr<t| d ƒ}t || d f¡S g }d}| D ]z}t|ƒdkrj|\}}t|ƒ}n|\}}}t t|ƒ|f¡}|dko¢|jtju o¢|j	du }|s¸| 
|||f¡ ||j7 }qHt|Ž \}	}
}dd„ |	D ƒ}t|Ž \}}	t |	|
|||dœ¡S )	ad  
        descr may be stored as dtype.descr, which is a list of (name, format,
        [shape]) tuples where format may be a str or a tuple.  Offsets are not
        explicitly saved, rather empty fields with name, format == '', '|Vn'
        are added as padding.  This function reverses the process, eliminating
        the empty padding fields.
        r   é   é   Ú Nc                 s   s$   | ]}t |tƒr|nd |fV  qd S ©N)Ú
isinstanceÚtuple)Ú.0Ún© r   úe/mounts/lovelace/software/anaconda3/envs/metadmg/lib/python3.9/site-packages/pyarrow/serialization.pyÚ	<genexpr>I   ó    z!descr_to_dtype.<locals>.<genexpr>)ÚnamesÚformatsÚtitlesÚoffsetsÚitemsize)r
   ÚstrÚnpÚdtyper   r   ÚlenÚtypeÚvoidr   Úappendr   Úzip)ÚdescrÚdtÚfieldsÚoffsetÚfieldÚnameZ	descr_strÚshapeZis_padr   r   r   Znametupsr   r   r   r   r   %   s.    





ÿr   c                 C   s   d  | ¡}tj|tdd� d S )Nz„'pyarrow.{}' is deprecated as of 2.0.0 and will be removed in a future version. Use pickle or the pyarrow IPC functionality instead.é   )Ú
stacklevel)ÚformatÚwarningsÚwarnÚFutureWarning)r$   Úmsgr   r   r   Ú_deprecate_serializationO   s
    ÿýr-   c                 C   sT   | j jdkr8| jjst | ¡} |  d¡tjj 	| j ¡fS |  
¡ tjj 	| j ¡fS d S ©Nú|OÚuint8)r   r   ÚflagsÚc_contiguousr   ÚascontiguousarrayÚviewÚlibr(   Údtype_to_descrÚtolist©Úobjr   r   r   Ú_serialize_numpy_array_list\   s
    
r:   c                 C   sV   | d dkr6| d j tjks J ‚| d  t| d ƒ¡S tj| d t  | d ¡d�S d S )Nr   r/   r   )r   )r   r   r0   r4   r   Úarray©Údatar   r   r   Ú_deserialize_numpy_array_listg   s    r>   c                 C   sZ   | j jdkr<| jjs t | j¡} | j d¡tjj	 
| j ¡fS | j ¡ tjj	 
| j ¡fS d S r.   )r   r   r1   r2   r   r3   ÚAr4   r5   r(   r6   r7   r8   r   r   r   Ú_serialize_numpy_matrixo   s
    r@   c                 C   sb   | d dkr@| d j tjks J ‚tj| d  t| d ƒ¡dd�S tj| d t  | d ¡dd�S d S )Nr   r/   r   F)Úcopy)r   rA   )r   r   r0   Úmatrixr4   r   r<   r   r   r   Ú_deserialize_numpy_matrixz   s    ÿrC   c                 C   sJ   t  ¡ }t j|| jd��}| | ¡ W d   ƒ n1 s80    Y  | ¡ S ©N)Úschema)ÚpaÚBufferOutputStreamÚRecordBatchStreamWriterrE   Zwrite_batchÚgetvalue)ÚbatchÚoutput_streamÚwrr   r   r   Ú_serialize_pyarrow_recordbatch†   s    (rM   c                 C   s6   t  | ¡�}| ¡ W  d   ƒ S 1 s(0    Y  d S r	   )rF   ÚRecordBatchStreamReaderZread_next_batch©ÚbufÚreaderr   r   r   Ú _deserialize_pyarrow_recordbatch�   s    rR   c                 C   s   t j | gdg¡}t|ƒS )Nr   )rF   ÚRecordBatchZfrom_arraysrM   )r;   rJ   r   r   r   Ú_serialize_pyarrow_array•   s    rT   c                 C   s   t | ƒ}|jd S ©Nr   )rR   Úcolumns)rP   rJ   r   r   r   Ú_deserialize_pyarrow_array›   s    rW   c                 C   sJ   t  ¡ }t j|| jd��}| | ¡ W d   ƒ n1 s80    Y  | ¡ S rD   )rF   rG   rH   rE   Zwrite_tablerI   )ÚtablerK   rL   r   r   r   Ú_serialize_pyarrow_table¤   s    (rY   c                 C   s6   t  | ¡�}| ¡ W  d   ƒ S 1 s(0    Y  d S r	   )rF   rN   Zread_allrO   r   r   r   Ú_deserialize_pyarrow_table«   s    rZ   c                 C   s   t j| t jd�}t|ƒS )N)Úprotocol)r   ÚdumpsÚHIGHEST_PROTOCOLr   )ÚxZpickledr   r   r   Ú_pickle_to_buffer°   s    r_   c                 C   s   t | ƒ}t |¡S r	   )Ú
memoryviewr   Úloads)r=   Zas_memoryviewr   r   r   Ú_load_pickle_from_bufferµ   s    rb   c                    s4  zdd l ‰W n ty    Y d S 0 dd lm‰ d‰‡‡‡fdd„‰‡fdd„‰ ‡‡‡‡fdd„}‡ fd	d
„}| jˆjd||d� | jˆjdttd� t	ˆj
dƒ�rt	ˆj
jdƒrÊ| jˆj
jjjdttd� t	ˆj
jdƒrò| jˆj
jjjdttd� t	ˆj
jdƒ�r| jˆj
jjjdttd� | jˆjdˆˆ d� d S )Nr   z´{0} serialization is not supported.
Note that {0} is planned to be deprecated in pandas future releases.
See https://github.com/pandas-dev/pandas/issues/19239 for more information.c                    s,   ˆj jr"t| ˆ jƒr"tˆ d¡ƒ‚ˆ | ¡S )NÚSparseDataFrame)Ú_pandas_apiÚ
has_sparser
   rc   ÚNotImplementedErrorr(   Zdataframe_to_serialized_dictr8   )ÚpdÚpdcompatÚsparse_type_error_msgr   r   Ú_serialize_pandas_dataframeÏ   s    
ÿÿzE_register_custom_pandas_handlers.<locals>._serialize_pandas_dataframec                    s
   ˆ   | ¡S r	   )Zserialized_dict_to_dataframer<   )rh   r   r   Ú_deserialize_pandas_dataframeØ   s    zG_register_custom_pandas_handlers.<locals>._deserialize_pandas_dataframec                    s6   ˆj jr"t| ˆjƒr"tˆ d¡ƒ‚ˆ ˆ | j| i¡ƒS )NÚSparseSeries)rd   re   r
   rl   rf   r(   Ú	DataFramer$   r8   )rj   rg   rh   ri   r   r   Ú_serialize_pandas_seriesÛ   s    
ÿÿzB_register_custom_pandas_handlers.<locals>._serialize_pandas_seriesc                    s   ˆ | ƒ}||j d  S rU   )rV   )r=   Zdeserialized)rk   r   r   Ú_deserialize_pandas_seriesä   s    zD_register_custom_pandas_handlers.<locals>._deserialize_pandas_seriesz	pd.Series©Zcustom_serializerZcustom_deserializerzpd.IndexÚarraysÚintervalz%pd.core.arrays.interval.IntervalArrayÚperiodz!pd.core.arrays.period.PeriodArrayÚ	datetimesz&pd.core.arrays.datetimes.DatetimeArrayzpd.DataFrame)ZpandasÚImportErrorZpyarrow.pandas_compatZpandas_compatÚregister_typeZSeriesÚIndexr_   rb   ÚhasattrÚcorerq   rr   ZIntervalArrayrs   ZPeriodArrayrt   ZDatetimeArrayrm   )Úcontextrn   ro   r   )rk   rj   rg   rh   ri   r   Ú _register_custom_pandas_handlers½   s`    ÿ		ýý
ü
ü
üýr{   c              	      s†   t dƒ zfdd l‰ dd„ }‡ fdd„}ˆ jˆ jˆ jˆ jˆ jˆ jˆ jˆ j	ˆ j
f	D ]}| j|d|j ||d� qNW n ty€   Y n0 d S )	NÚ%register_torch_serialization_handlersr   c                 S   sH   | j r8tjj|  ¡  ¡  ¡ |  ¡  ¡  ¡ jt	| j
ƒd�S |  ¡  ¡ S d S )N)r%   )Z	is_sparserF   ÚSparseCOOTensorÚ
from_numpyZ_valuesÚdetachÚnumpyZ_indicesÚTÚlistr%   r8   r   r   r   Ú_serialize_torch_tensor  s    ýzFregister_torch_serialization_handlers.<locals>._serialize_torch_tensorc                    sJ   t | tjƒr<ˆ j|  ¡ d j|  ¡ d d d …df | jd�S ˆ  | ¡S d S )Nr   r   )ÚindicesÚvaluesÚsize)r
   rF   r}   Zsparse_coo_tensorÚto_numpyr�   r%   r~   r<   ©Útorchr   r   Ú_deserialize_torch_tensor  s    ýzHregister_torch_serialization_handlers.<locals>._deserialize_torch_tensorztorch.rp   )r-   r‰   ZFloatTensorZDoubleTensorZ
HalfTensorZ
ByteTensorZ
CharTensorZShortTensorZ	IntTensorZ
LongTensorZTensorrv   Ú__name__ru   )Úserialization_contextrƒ   rŠ   Útr   rˆ   r   r|     s     		þ
ýr|   c           	      C   s”   dd„ }dd„ }| j tjd||d� dd„ }d	d
„ }| j tjd||d� dd„ }dd„ }| j tjd||d� dd„ }dd„ }| j tjd||d� d S )Nc                 S   s   t | ƒS r	   )r‚   r8   r   r   r   Ú_serialize_deque5  s    zF_register_collections_serialization_handlers.<locals>._serialize_dequec                 S   s
   t  | ¡S r	   )ÚcollectionsÚdequer<   r   r   r   Ú_deserialize_deque8  s    zH_register_collections_serialization_handlers.<locals>._deserialize_dequezcollections.dequerp   c                 S   s   t |  ¡ ƒt |  ¡ ƒfS r	   ©r‚   Úkeysr…   r8   r   r   r   Ú_serialize_ordered_dict@  s    zM_register_collections_serialization_handlers.<locals>._serialize_ordered_dictc                 S   s   t  t| d | d ƒ¡S ©Nr   r   )r�   ÚOrderedDictr   r<   r   r   r   Ú_deserialize_ordered_dictC  s    zO_register_collections_serialization_handlers.<locals>._deserialize_ordered_dictzcollections.OrderedDictc                 S   s   t |  ¡ ƒt |  ¡ ƒ| jfS r	   )r‚   r“   r…   Údefault_factoryr8   r   r   r   Ú_serialize_default_dictK  s    zM_register_collections_serialization_handlers.<locals>._serialize_default_dictc                 S   s   t  | d t| d | d ƒ¡S )Nr   r   r   )r�   Údefaultdictr   r<   r   r   r   Ú_deserialize_default_dictN  s    zO_register_collections_serialization_handlers.<locals>._deserialize_default_dictzcollections.defaultdictc                 S   s   t |  ¡ ƒt |  ¡ ƒfS r	   r’   r8   r   r   r   Ú_serialize_counterV  s    zH_register_collections_serialization_handlers.<locals>._serialize_counterc                 S   s   t  tt| d | d ƒƒ¡S r•   )r�   ÚCounterÚdictr   r<   r   r   r   Ú_deserialize_counterY  s    zJ_register_collections_serialization_handlers.<locals>._deserialize_counterzcollections.Counter)rv   r�   r�   r–   rš   r�   )	rŒ   rŽ   r‘   r”   r—   r™   r›   rœ   rŸ   r   r   r   Ú,_register_collections_serialization_handlers4  s8    ýýýýr    c                    s�   zxddl m}m}m}m‰m‰m‰m‰  ‡ ‡‡‡fdd„}dd„ }| j|d||d� | j|d	||d� | j|d
||d� W n t	yŠ   Y n0 d S )Nr   )Ú
csr_matrixÚ
csc_matrixÚ
coo_matrixÚisspmatrix_cooÚisspmatrix_csrÚisspmatrix_cscÚ
isspmatrixc                    sz   ˆ| ƒrdt j | ¡fS ˆ| ƒr0dt j | ¡fS ˆ| ƒrHdt j | ¡fS ˆ | ƒrddt j |  ¡ ¡fS td | d ¡ƒ‚d S )NÚcooÚcsrÚcscú%Serialization of {} is not supported.r   )rF   r}   Z
from_scipyZSparseCSRMatrixZSparseCSCMatrixZto_coorf   r(   r8   ©r§   r¤   r¦   r¥   r   r   Ú_serialize_scipy_sparsel  s    ÿz9_register_scipy_handlers.<locals>._serialize_scipy_sparsec                 S   sX   | d dkr| d   ¡ S | d dkr0| d   ¡ S | d dkrH| d   ¡ S | d   ¡ S d S )Nr   r¨   r   r©   rª   )Zto_scipyr<   r   r   r   Ú_deserialize_scipy_sparse}  s    z;_register_scipy_handlers.<locals>._deserialize_scipy_sparsezscipy.sparse.coo.coo_matrixrp   zscipy.sparse.csr.csr_matrixzscipy.sparse.csc.csc_matrix)
Zscipy.sparser¡   r¢   r£   r¤   r¥   r¦   r§   rv   ru   )rŒ   r¡   r¢   r£   r­   r®   r   r¬   r   Ú_register_scipy_handlersf  s*    $ýýý
r¯   c                    sP   z8dd l ‰ ‡ fdd„}‡ fdd„}| jˆ jd||d� W n tyJ   Y n0 d S )Nr   c                    s0   t | ˆ jƒrdtj | ¡fS td ˆ j¡ƒ‚d S )Nr¨   r«   )r
   ÚCOOrF   r}   Zfrom_pydata_sparserf   r(   r8   ©Úsparser   r   Ú_serialize_pydata_sparse¥  s
    
ÿzB_register_pydata_sparse_handlers.<locals>._serialize_pydata_sparsec                    sD   | d dkr@| d   ¡ \}}ˆ j|d d …df |j| d jd�S d S )Nr   r¨   r   )r=   Úcoordsr%   )r‡   r°   r�   r%   )r=   Z
data_arrayr´   r±   r   r   Ú_deserialize_pydata_sparse¬  s    þzD_register_pydata_sparse_handlers.<locals>._deserialize_pydata_sparsez
sparse.COOrp   )r²   rv   r°   ru   )rŒ   r³   rµ   r   r±   r   Ú _register_pydata_sparse_handlers¡  s    ý
r¶   c                 C   sÊ   | j tddd„ dd„ d� | j tdd„ ƒddd	� | j td
dd	� | j tjdttd� | j tjdtt	d� | j t
jdttd� | j t
jdttd� | j t
jdttd� t| ƒ t| ƒ t| ƒ t| ƒ d S )NÚintc                 S   s   t | ƒS r	   )r   r8   r   r   r   Ú<lambda>Ç  r   z:_register_default_serialization_handlers.<locals>.<lambda>c                 S   s   t | ƒS r	   )r·   r<   r   r   r   r¸   È  r   rp   c                   S   s   dS rU   r   r   r   r   r   r¸   Ë  r   ÚfunctionT)Úpickler   z	np.matrixznp.arrayzpyarrow.Arrayzpyarrow.RecordBatchzpyarrow.Table)rv   r·   r   r   rB   r@   rC   Zndarrayr:   r>   rF   ZArrayrT   rW   rS   rM   rR   ÚTablerY   rZ   r    r{   r¯   r¶   ©rŒ   r   r   r   Ú(_register_default_serialization_handlers½  sN    ýþýýýýýr½   c                 C   s   t dƒ t| ƒ d S )NÚ'register_default_serialization_handlers)r-   r½   r¼   r   r   r   r¾   ï  s    r¾   c                  C   s   t dƒ tƒ } t| ƒ | S )NÚdefault_serialization_context)r-   r   r½   )rz   r   r   r   r¿   ô  s    r¿   )#r�   r)   r€   r   ZpyarrowrF   Zpyarrow.libr   r   r   Zcloudpickleru   Znumpy.lib.formatr   r-   r:   r>   r@   rC   rM   rR   rT   rW   rY   rZ   r_   rb   r{   r|   r    r¯   r¶   r½   r¾   r¿   r   r   r   r   Ú<module>   sB   
*	Q&2;2