
    Lg>3                    l   U d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlZd dlZd dlmZ d dlmZ d dlmZ d dlmZ d dlZd dlmZmZ d dlmZ d d	lmZ d d
lmZ d dlm Z  d dl!m"Z"  G d de#      Z$d-dZ% e
jL                         Z'i a(de)d<    ed      a*de)d<   dd	 	 	 	 	 	 	 d.dZ+d/dZ,e-e.e/e0 e1d      e2e3 e1e4      ejj                  ejl                  ejn                  ej                  ejp                  ejr                  fZ: e"       Z;e;jy                  e:e       e;jy                  ejz                  e>f      d        Z?e;jy                  e      d        Z@e;jy                  eA      d        ZBd0dZCe;jy                  eDeEf      d        ZFe;jy                  e      d        ZGe;jy                  e      d        ZHe;jy                  eef      d        ZIe;jy                  ej                  ej                  f      d        ZLe;jy                  ej                        d         ZNe;jy                  eO      d!        ZP eA       ZQd1d"ZRd2d#ZSd$ ZTe;j                  d%      d&        ZVe;j                  d'      d(        ZWe;j                  d)      d*        ZXe;j                  d+      d,        ZYy)3    )annotationsN)OrderedDict)Iterable)
ContextVar)partial)curryidentity)Compose)config)literalhash_buffer_hex)Dispatchc                      e Zd Zy)TokenizationErrorN)__name__
__module____qualname__     -lib/python3.12/site-packages/dask/tokenize.pyr   r      s    r   r   c                     t        |       }|r$|t        t        |j                                     f}t        j                  t        |      j                         d      j                         S )NF)usedforsecurity)_normalize_seq_funcsorteditemshashlibmd5strencode	hexdigest)argskwargstokens      r   	_tokenizer%   !   sS    '-E*6&,,.+ABB ;;s5z((*EBLLNNr   zdict[int, tuple[int, object]]_SEEN_ENSURE_DETERMINISTICzContextVar[bool | None])ensure_deterministicc                H   t         5  t        i c}ad}	 t        j                          	 t        |i ||rt        j                  |       |acddd       S # t        $ r t        j                  |       }Y Nw xY w# |rt        j                  |       |aw xY w# 1 sw Y   yxY w)a  Deterministic token

    >>> tokenize([1, 2, '3'])  # doctest: +SKIP
    '06961e8de572e73c2e74b51348177918'

    >>> tokenize('Hello') == tokenize('Hello')
    True

    Parameters
    ----------
    args, kwargs:
        objects to tokenize
    ensure_deterministic: bool, optional
        If True, raise TokenizationError if the objects cannot be deterministically
        tokenized, e.g. two identical objects will return different tokens.
        Defaults to the `tokenize.ensure-deterministic` configuration parameter.
    N)tokenize_lockr&   r'   getLookupErrorsetr%   reset)r(   r"   r#   seen_beforer$   s        r   tokenizer0   /   s    * 
"BU	D!%%'	 d-f-%++E2E 

  	D)--.BCE	D
 %++E2E 
s>   BA
A:BA74B6A77B:BBB!c                    	 t         j                         }|s|!t        j                  d      rt	        |       y y # t        $ r d }Y 3w xY w)Nztokenize.ensure-deterministic)r'   r+   r,   r   r   )msgvals     r   _maybe_raise_nondeterministicr4   S   sS    #'') ckfjj)HI$$ Jk  s   < A
	A
c                j   t        |       t        v rdt        t        |          d   fS t        t              | ft        t        |       <   	 dt        t	        | j                         d             ft        j                  t        |       d        S # t        j                  t        |       d        w xY w)N__seenr   dictc                    t        | d         S Nr   )hash)kvs    r   <lambda>z normalize_dict.<locals>.<lambda>z   s    T"Q%[r   key)idr&   lenr   r   r   popds    r   normalize_dictrD   s   s    	!u~r!ua((u:q=E"Q%L*1779"89
 
 			"Q%		"Q%s   &B !B2c                \    t        t        |       t        | j                               f      S N)r   typelistr   rB   s    r   normalize_ordered_dictrI      s     Qaggi9::r   c                :    dt        t        | t                    fS )Nr-   r=   )r   r   r   ss    r   normalize_setrM      s    
 %fQC&8999r   c                   d }t        |       t        v rdt        t        |          d   fS t        t              | ft        t        |       <   	 t        t	        ||             t        t        |       = S # t        t        |       = w xY w)Nc                <    t        | t              r| S t        |       S rF   )
isinstance_IDENTITY_DISPATCHnormalize_token)items    r   _inner_normalize_tokenz3_normalize_seq_func.<locals>._inner_normalize_token   s    d./Kt$$r   r6   r   )r?   r&   r@   tuplemap)seqrT   s     r   r   r      sn    % 
#w%r#w***Z_E"S'NS/56"S'NE"S'Ns   A3 3Bc                B    t        |       j                  t        |       fS rF   )rG   r   r   )rW   s    r   normalize_seqrY      s    923777r   c                &    dt         |              fS )Nr   )rR   )lits    r   normalize_literalr\      s    oce,,,r   c                H    t        | j                  f| j                  z         S rF   )r   firstfuncsfuncs    r   normalize_composerb      s    

}tzz9::r   c                Z    t        | j                  | j                  | j                  f      S rF   )r   ra   r"   keywordsr`   s    r   normalize_partialre      s     		499dmmDEEr   c                D    t        | j                        | j                  fS rF   )rR   __self__r   )meths    r   normalize_bound_methodri      s    4==)4==88r   c                v    t        | dd       }| t        j                  |      st        |       S t	        |       S )Nrg   )getattrinspectismoduleri   normalize_object)ra   selfs     r   $normalize_builtin_function_or_methodrp      s;     4T*D 0 0 6%d++%%r   c                   t        | dd       }|t        | t              s |       S t        |       t        u rt	        |       S t        | t              rt        j                  |        t        j                  |       rt        | t              st        |       S 	 t        |       S # t        $ r0 t        d| d       t        j                         j                  cY S w xY w)N__dask_tokenize__zObject z cannot be deterministically hashed. See https://docs.dask.org/en/latest/custom-collections.html#implementing-deterministic-hashing for more information.)rk   rP   rG   object_normalize_pure_objectcopyreg
_slotnamesdataclassesis_dataclass_normalize_dataclass_normalize_pickle	Exceptionr4   uuiduuid4hex)omethods     r   rn   rn      s    Q+T2F*Q"5xAw&%a((!T1":a+>#A&&  ##  %aU $ $	

 zz| s   
B$ $6CCc                \    t        d       t        j                  |        dt        |       fS )Nzobject() cannot be deterministically hashed. See https://docs.dask.org/en/latest/custom-collections.html#implementing-deterministic-hashing for more information.rs   )r4   _seen_objectsaddr?   )r   s    r   rt   rt      s-    !	  aRU?r   c                   g }d }t         t        fD ]t  }t        d      D ]X  }|j                          	 |j	                  | d|j
                        }|j                  ||       t        |      }||k(  r e|}Z t        d        n |(t        d       t        t        j                               }||D cg c]  }t        |       c}fS # t        $ r Y  w xY wc c}w )N      )protocolbuffer_callback)buffersz$Failed to tokenize deterministically)picklecloudpicklerangecleardumpsappendloadsr   r{   r4   intr|   r}   )r   r   pikmod_outpik2bufs           r   rz   rz      s    )+GC $qAMMOiiAw~~iN		#w	/&s+ d{C  **PQ % {%&LM$**,9#%999   :s   <C7C	CCc                >   t        j                  |       D cg c].  }|j                  t        t	        | |j                  d             f0 }}| j
                  }|j                  D cg c]  }|t	        ||      f }}t        t        |             ||fS c c}w c c}w rF   )	rw   fieldsnamerR   rk   __dataclass_params__	__slots__rn   rG   )objfieldr   paramsattrs        r   ry   ry     s     !'',,E 
_WS%**d%CDE,   %%F8>8H8HI8HtWVT*+8HFIDI&66
 Js   3B'Bpandasc                 ^   dd l t        j                  j                        d        } t        j                  j                        d        } t        j                  j
                        d        }t        j                  j                  j                        t        j                  j                  j                        t        j                  j                  j                        d                      }t        j                  j                  j                        d        }t        j                  j                        d        }t        j                  j                        d        }t        j                  j                  j                        d	        }t        j                  j                  j                  j                         d
        }t        j                  j                  j"                  j$                        d        }t        j                  j                  j                  j&                        d        }t        j                  t)        j*                              fd       }	t        j                  j,                  j.                        d        }
y )Nr   c                H    | j                   }| j                  t        |      gS rF   )arrayr   rR   )indvaluess     r   normalize_indexz(register_pandas.<locals>.normalize_index  s    /&122r   c                    | j                   }| j                  g| j                  D cg c]  }t        |       c}z   |D cg c]  }t        |       c}z   S c c}w c c}w rF   )codesr   levelsrR   )r   r   xs      r   r   z(register_pandas.<locals>.normalize_index  s[    		XXJ+.::6:aq!:67+015aq!512	
61s   AAc                V    t        | j                        t        | j                        gS rF   )rR   r   dtype)cats    r   normalize_categoricalz.register_pandas.<locals>.normalize_categorical  s    		*OCII,FGGr   c                V    t        | j                        t        | j                        gS rF   )rR   asi8r   arrs    r   normalize_period_arrayz/register_pandas.<locals>.normalize_period_array#  s!      )?399+EFFr   c                ~    t        | j                        t        | j                        t        | j                        gS rF   )rR   leftrightclosedr   s    r   normalize_interval_arrayz1register_pandas.<locals>.normalize_interval_array)  s2     CHH%CII&CJJ'
 	
r   c                    | j                   | j                  t        | j                        t        | j                        gS rF   )r   r   rR   _valuesindexrK   s    r   normalize_seriesz)register_pandas.<locals>.normalize_series1  s4     FFGGAII&AGG$	
 	
r   c                    | j                   }t        |j                        | j                  | j                  gz   }t        t        t        |            S rF   )_mgrrH   arrayscolumnsr   rV   rR   )dfmgrdatas      r   normalize_dataframez,register_pandas.<locals>.normalize_dataframe:  s=    ggCJJ2::rxx"88C.//r   c                    	 t        |       t        | j                        fS # t        $ r# t        |       t        | j                        fcY S w xY wrF   )rG   rR   	_pa_arrayAttributeError_datar   s    r   normalize_extension_arrayz2register_pandas.<locals>.normalize_extension_array@  sF    	;Is}}=>> 	;Isyy9::	;s   " )AAc                >    dd l }t        |j                  |             S r9   )numpyrR   asarray)r   nps     r   r   z2register_pandas.<locals>.normalize_extension_arrayG  s    rzz#//r   c                V    t        | j                        t        | j                        gS rF   )rR   
categoriesorderedr   s    r   normalize_categorical_dtypez4register_pandas.<locals>.normalize_categorical_dtypeN  s!     0 01?5==3QRRr   c                ,    t        | j                        S rF   )rR   r   r   s    r   normalize_period_dtypez/register_pandas.<locals>.normalize_period_dtypeR  s    uzz**r   c                    j                   S rF   )NA)napds    r   normalize_naz%register_pandas.<locals>.normalize_naV  s    uur   c                    | j                   S rF   )freqstr)offsets    r   normalize_offsetz)register_pandas.<locals>.normalize_offsetZ  s    ~~r   )r   rR   registerIndex
MultiIndexCategoricalr   PeriodArrayDatetimeArrayTimedeltaArrayIntervalArraySeries	DataFrameArrowExtensionArrayapi
extensionsExtensionArraytypesCategoricalDtypeExtensionDtyperG   r   offsets
BaseOffset)r   r   r   r   r   r   r   r   r   r   r   r   s              @r   register_pandasr     sK   bhh'3 (3 bmm,
 -
 bnn-H .H bii334bii556bii667G 8 7 5G bii556
 7
 bii(
 )
 bll+0 ,0
 bii;;<; =; bff//>>?0 @0 bffll;;<S =S bff//>>?+ @+ d255k* + bjj334 5r   numbac                 ~    dd l } t        j                  | j                  j                  j
                        d        }y )Nr   c                T    t        | j                         | j                         f      S rF   )rR   _reduce_class_reduce_states)r   s    r   normalize_numba_ufuncz-register_numba.<locals>.normalize_numba_ufuncc  s$     1 1 3S5G5G5IJKKr   )r   rR   r   core	serializeReduceMixin)r   r   s     r   register_numbar   _  s5    ejj22>>?L @Lr   pyarrowc                 ~   dd l } t        j                  | j                        d        }t        j                  | j                        d        }t        j                  | j
                        d        }t        j                  | j                        d        }t        j                  | j                        d        }y )Nr   c                0    t        j                  | d      S )N   )r   )r   r   dts    r   normalize_datatypez,register_pyarrow.<locals>.normalize_datatypel  s    ||B++r   c                X    dt        | j                        t        | j                        fS )Nzpa.Table)rR   schemar   r  s    r   normalize_tablez)register_pyarrow.<locals>.normalize_tablep  s)     BII&BJJ'
 	
r   c                X    dt        | j                        t        | j                        fS )Nzpa.ChunkedArray)rR   rG   chunksr   s    r   normalize_chunked_arrayz1register_pyarrow.<locals>.normalize_chunked_arrayx  s)     CHH%CJJ'
 	
r   c                `    dt        | j                        t        | j                               fS )Nzpa.Array)rR   rG   r   r   s    r   r	  z1register_pyarrow.<locals>.normalize_chunked_array  s+     CHH%CKKM*
 	
r   c                    dt        |       fS )Nz	pa.Bufferr   )r   s    r   r	  z1register_pyarrow.<locals>.normalize_chunked_array  s    _S122r   )r   rR   r   DataTypeTableChunkedArrayArrayBuffer)par  r  r	  s       r   register_pyarrowr  h  s    bkk*, +, bhh'
 (
 boo.
 /
 bhh'
 (
 bii(3 )3r   r   c                 :   dd l t        j                  j                        d        } t        j                  j                        fd       }t        j                  j
                        d        }t        j                  j                        d        }y )Nr   c                   | j                   s| j                         | j                  fS | j                  j                  r8	 	 t	        dj                  | j                        j                  dd            }n+	 t	        | j                  d      j                  d            }|| j                  | j                   fS # t        $ r' t	        dj                  | j                              }Y tw xY w# t        t        f$ r t        |       cY S w xY w# t        t        t        f$ r; t	        | j!                         j                  d      j                  d            }Y w xY w)	N-zutf-8surrogatepass)encodingerrors   -K)orderi1)shaperS   r   	hasobjectr   joinflatr    UnicodeDecodeError	TypeErrorrn   ravelviewBufferErrorr   
ValueErrorcopy)r   r   s     r   normalize_arrayz'register_numpy.<locals>.normalize_array  s*   wwFFHagg&&77+	>*(//%,_ 0 DM&qwwSw'9'>'>t'DE aggqww'' * >*499QVV+<=D> 12 +'**+
  < M&qvvx~~C~'@'E'Ed'KLMs=   5B; 8*D ;-C+(C. *C++C. .D
DAEEc                8    t        j                  |             S rF   )r   ascontiguousarray)mmr   s    r   normalize_mmapz&register_numpy.<locals>.normalize_mmap  s    r33B788r   c                    	 t        |       S # t        $ r0 t        d| d       t        j                         j
                  cY S w xY w)NzCannot tokenize numpy ufunc z. Please use functions of the dask.array.ufunc module instead. See also https://docs.dask.org/en/latest/array-numpy-compatibility.html)rz   r{   r4   r|   r}   r~   r`   s    r   normalize_ufuncz'register_numpy.<locals>.normalize_ufunc  sP    	$$T** 	$).th 7Q Q
 ::<###	$s   
 6AAc                    | j                   S rF   )r   r   s    r   normalize_dtypez'register_numpy.<locals>.normalize_dtype  s    yyr   )r   rR   r   ndarraymemmapufuncr   )r(  r,  r.  r0  r   s       @r   register_numpyr4    s    bjj)( *(0 bii(9 )9 bhh'	$ (	$ bhh' (r   )r"   rs   r#   rs   returnr   )r"   rs   r(   zbool | Noner#   rs   r5  r   )r2   r   r5  None)rW   zIterable[object]r5  ztuple[object, ...])r   rs   r5  ztuple[str, int])r   rs   r5  rU   )Z
__future__r   ru   rw   datetimedecimalr   rl   pathlibr   	threadingr   r|   collectionsr   collections.abcr   contextvarsr   	functoolsr   r   tlzr   r	   tlz.functoolzr
   daskr   	dask.corer   dask.hashingr   
dask.utilsr   RuntimeErrorr   r%   RLockr*   r&   __annotations__r'   r0   r4   r   floatr   bytesrG   slicecomplexEllipsisDecimaldatetime	timedeltaPurePathrQ   rR   r   MappingProxyTyper7   rD   rI   r-   rM   r   rU   rH   rY   r\   rb   re   
MethodTypeMethodWrapperTyperi   BuiltinFunctionTyperp   rs   rn   r   rt   rz   ry   register_lazyr   r   r  r4  r   r   r   <module>rX     s3   "            # $ "    !   ( 	 	O  	!')$ )1;<S1T . T 8<! ! )4! GM! ! H% 		J	NOOMMMM   *    511489	 :	 +&; '; #: :  5$-(8 )8 '"- #- '"; #; 7E*+F ,F 5++U-D-DEF9 G9 %334& 5& &!  " 2 :47 x(N )Nb w'L (L y)!3 *!3H w'. (.r   