
    Cd}$                    R   d dl mZ d dlZd dlZ ej        d          Zd dlZd dlmZ d dlm	Z	m
Z
 d dlmZmZ d dlmZ d dlmZmZmZ  G d	 d
e
          Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zej         !                    d          ej         "                    dedfedfedfedfedfedfedfedfg          ej         "                    dddg          d                                     Z#d Z$d  Z%ej         "                    d!e$e%g          d"             Z&d# Z'dS )$    )annotationsNdistributed)getitem)ClientSchedulerPlugin)clusterloop)HighLevelGraph)ArrayChunkShapeDepArraySliceDepfractional_slicec                  "    e Zd ZdZdZd Zd ZdS )SchedulerImportCheckzAPlugin to help record which modules are imported on the schedulerzimport-checkc                    || _         d S N)pattern)selfr   s     6lib/python3.11/site-packages/dask/tests/test_layers.py__init__zSchedulerImportCheck.__init__   s        c                  K   t                      | _        t          t          j                  D ]V}|                    | j                  s| j                            |           7t          j                            |           Wd S r   )setstart_modulessysmodules
startswithr   addpop)r   	schedulermods      r   startzSchedulerImportCheck.start   s~       UUs{## 	% 	%C>>$,// %"&&s++++ $$$$	% 	%r   N)__name__
__module____qualname____doc__namer   r!    r   r   r   r      s=        KKD  % % % % %r   r   c                 (   t          j        d          } ddt          fdD                       }|                     |          }t	          |          fd}t                    D ] }t                    D ]} |||           !d S )Ndask.array.core   r*      c              3  "   K   | ]	}|z  V  
d S r   r'   .0nds     r   	<genexpr>z-test_array_chunk_shape_dep.<locals>.<genexpr>*   '      ''A!a%''''''r   c                ,    | |f         }|k    sJ d S r   r'   )ijchunk_shape
array_depschunks      r   checkz)test_array_chunk_shape_dep.<locals>.check.   s(     !Q(e######r   )pytestimportorskiptuplenormalize_chunksr   range	dacshapechunksr:   r5   r6   r8   r9   r1   s	         @@@r   test_array_chunk_shape_deprD   &   s    

/
0
0C	AE'''''''''E!!%//F#F++J$ $ $ $ $ $ 1XX  q 	 	AE!QKKKK	 r   c                 (   t          j        d          } ddt          fdD                       }|                     |          }t	          |          fd}t                    D ] }t                    D ]} |||           !d S )Nr)   r*   r+   c              3  "   K   | ]	}|z  V  
d S r   r'   r.   s     r   r2   z(test_array_slice_deps.<locals>.<genexpr>;   r3   r   c                    | |f         }|t          d         | z  d         | dz   z  d           t          d         |z  d         |dz   z  d           fk    sJ d S )Nr      )slice)r5   r6   slicesr8   r9   s      r   r:   z$test_array_slice_deps.<locals>.check?   s    QF#%(Q,aAE 2D99%(Q,aAE 2D99
 
 
 
 
 
 
r   )r;   r<   r=   r>   r   r?   r@   s	         @@@r   test_array_slice_depsrK   7   s    

/
0
0C	AE'''''''''E!!%//Fv&&J
 
 
 
 
 
 1XX  q 	 	AE!QKKKK	 r   c                   t          j        d          }t          j        d          }|                    t          d          t          dd          d          }|                    |d                              dd	
          S )Npandasdask.dataframe
      abr*   npartitionsrR   tasksshuffle)r;   r<   	DataFramer?   from_pandasrX   tmpdirpddddfs       r   _dataframe_shuffler`   K   su    		X	&	&B		-	.	.B 
E"IIE"bMM::	;	;B>>"!>,,44S'4JJJr   c                
   t          j        d          }t          j        d          }|                    t          d          t          dd          d          }|                    |d                                          S )NrM   rN   rO   rP   rQ   r*   rT   )r;   r<   rY   r?   rZ   meanr[   s       r   _dataframe_tree_reductionrc   T   sn    		X	&	&B		-	.	.B 
E"IIE"bMM::	;	;B>>"!>,,11333r   c                F   t          j        d          }t          j        d          }|                    t          d          t          dd          d          }|                    |d          }|                    |d          }|                    |d	d
d          S )NrM   rN   rO   rP   rQ      rT   rH   leftTrV   )how	broadcastrX   )r;   r<   rY   r?   rZ   merge)r\   r]   r^   r_   ddf1ddf2s         r   _dataframe_broadcast_joinrl   ]   s    		X	&	&B		-	.	.B 
E"IIE"bMM::	;	;B>>"!>,,D>>"!>,,D::d$:HHHr   c                    t          j        d          }|                    d          |                    d          z   S )N
dask.arrayd   )r;   r<   oneszeros)r\   das     r   _array_creationrt   h   s5    		\	*	*B 776??RXXf----r   c                    t          j        d          }|                    d          }|                    d dd          S )Nrn   ro   c                    | S r   r'   xs    r   <lambda>z$_array_map_overlap.<locals>.<lambda>r   s    q r   rH   none)depthboundary)r;   r<   rq   map_overlap)r\   rs   arrays      r   _array_map_overlapr   o   s>    		\	*	*BGGFOOE[[FCCCr   c                    t          dddi          t          dt          dd          ffk    sJ t          dddd          t          dt          d d d           t          d	d           ffk    sJ t          d
ddd          t          dt          dd          t          d	d           ffk    sJ t          dddi          } t          | d         d         t                    sJ d S )N)rx   g@r   r*   )rx      )rx   r,   ffffff@r,   )r   rH   )rx   r,   r   )rx   g333333@r   rH   )r   r   rI   
isinstanceint)fss    r   test_fractional_slicer   u   s   JA//GXaQR~3VVVVVMqQ<<88	tT4	 	 %D//2=     Oq\\::	q!eBoo&?     
*q!f	-	-BbeAh$$$$$$$r   c                   t          j        d           t          j        d          }t          j        d          }	 dd lm} n# t          $ r d }Y nw xY w|                    |                    dt          d          i          d                              t          |           d	
           dg}|
                    t          |           d	|          }|r)|
                    t          |           d|          }||fS |S )Nzpyarrow.parquetrM   rN   r   rR   rO   r*   rT   pyarrowengine)rR   z<=r*   )r   filterszpyarrow-dataset)r;   r<   pyarrow.datasetdatasetImportErrorrZ   rY   r?   
to_parquetstrread_parquet)r\   r]   r^   pa_dsr   rj   rk   s          r   _pq_pyarrowr      s/   
)***		X	&	&B		-	.	.B'''''''    NN2<<eBii 011qNAALLF M     G??3v;;y'?JJD 
 KK$  
 

 d|s   A AAc                x   t          j        d           t          j        d          }t          j        d          }|                    |                    dt	          d          i          d                              t          |           d           |                    t          |           d          S )	NfastparquetrM   rN   rR   rO   r*   rT   r   )r;   r<   rZ   rY   r?   r   r   r   r\   r]   r^   s      r   _pq_fastparquetr      s    
&&&		X	&	&B		-	.	.BNN2<<eBii 011qNAALLF M    ??3v;;}?===r   c                   t          j        d          }t          j        d          }|                    |                    dt	          d          i          d                              t          |                      |                    t          j	        
                    t          |           d                    S )NrM   rN   rR   rO   r*   rT   *)r;   r<   rZ   rY   r?   to_csvr   read_csvospathjoinr   s      r   	_read_csvr      s    		X	&	&B		-	.	.BNN
c599%&&    fS[[;;rw||CKK55666r   z#8480)reasonzop,libzpandas.znumpy.optimize_graphTFc                   t          dt                    gi          5 \  }}t          |d         |          5 }|                     | |          |           |                    d           }|                    d           }	||	z
  }
t          fd|	D                       rJ t          fd	|
D                       rJ 	 d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )
Nplugins)scheduler_kwargsaddress)r	   )r   c                 4    t          t          j                  S r   )r   r   r   r'   r   r   ry   z>test_scheduler_highlevel_graph_unpack_import.<locals>.<lambda>   s    S5E5E r   c                :    | j         t          j                 j        S r   )r   r   r&   r   )dask_schedulers    r   ry   z>test_scheduler_highlevel_graph_unpack_import.<locals>.<lambda>   s    ~'=(-( r   c              3  B   K   | ]}|                               V  d S r   r   r/   modulelibs     r   r2   z?test_scheduler_highlevel_graph_unpack_import.<locals>.<genexpr>   s1      NNf6,,S11NNNNNNr   c              3  B   K   | ]}|                               V  d S r   r   r   s     r   r2   z?test_scheduler_highlevel_graph_unpack_import.<locals>.<genexpr>   s1      LLf6,,S11LLLLLLr   )r   r   r   computerun_on_schedulerany)opr   r   r	   r\   r   workerscend_modulesr   new_moduless    `         r   ,test_scheduler_highlevel_graph_unpack_importr      s   & 
9/CC/H/H.I"J	K	K	K M PIi(t444 	MIIbbjjI@@@ ,,-E-EFFK..    M
 &5K NNNNNNNNNNNN LLLLLLLLLLLLL%	M 	M 	M 	M 	M 	M 	M 	M 	M 	M 	M 	M 	M 	M 	M	M M M M M M M M M M M M M M M M M Ms5   C.BC
C.C	C.C	C..C25C2c                0    |                      dd          S )Nrx   rV   rW   rW   ddfs    r   _shuffle_opr      s    ;;sG;,,,r   c                V    |                      d                              ddi          S )Nr&   rx   rb   )groupbyaggr   s    r   _groupby_opr      s&    ;;v""C=111r   r   c                l   t          j        d           t          j        d          } | |                    d                                                    }|j        }|                    |                                          }|                                |                                k    sJ d S )NrN   dask.datasets
2000-01-15end)r;   r<   
timeseriescountdaskcull__dask_keys__get_all_dependencies)r   datasetsresultgraphculled_graphs        r   $test_dataframe_cull_key_dependenciesr      s     ()))"?33HR###5566<<>>FKE::f224455L%%''<+L+L+N+NNNNNNNr   c                    t          j        d          } t          j        d          }|                     d          }dd}i }t          |j                  D ]}d |j        |ff|||f<   d ||ff||f<    t          j        ||g	          }|j        	                    ||j
        |j                  }|j        }fd
|                                D             }|                    |          }	|	j                                        }
|	                                }|
|k    sJ |                                }t%          |                                          D ]"\  }|dk    r|                    |f           #||k    sJ d S )Nr   rN   r   r   custom_graph_testcustom_graph_test_0c                    | S r   r'   rw   s    r   ry   zCtest_dataframe_cull_key_dependencies_materialized.<locals>.<lambda>  s    a r   c                    | S r   r'   rw   s    r   ry   zCtest_dataframe_cull_key_dependencies_materialized.<locals>.<lambda>  s    A r   )dependenciesc                $    g | ]}|d fk    
|S )r   r'   )r/   kr&   s     r   
<listcomp>zEtest_dataframe_cull_key_dependencies_materialized.<locals>.<listcomp>  s"    GGGdAY1r   r   )r;   r<   r   r?   rU   _namer
   from_collectionscorenew_dd_object_meta	divisionsr   r   r   key_dependenciescopyr   listkeysr   )r   r^   r   name_0dskr5   r   r   culled_keysr   cached_depsdepsdeps0r&   s                @r   1test_dataframe_cull_key_dependencies_materializedr     s   
 "?33H		-	.	.B


,

/
/C D"F
C3?## 4 4'K#)Q8VQK%+{3T1I

)$3%
H
H
HCW""3ciGGFKE HGGGf2244GGGK::k**L /4466K,,..D$ &&((E

%% ! !a66IItQi    D======r   )(
__future__r   r   r;   r<   r   r   operatorr   r   r   distributed.utils_testr   r	   dask.highlevelgraphr
   dask.layersr   r   r   r   rD   rK   r`   rc   rl   rt   r   r   r   r   r   markxfailparametrizer   r   r   r   r   r'   r   r   <module>r      s   " " " " " " 				 !f!-00 



       / / / / / / / / 0 0 0 0 0 0 0 0 . . . . . . K K K K K K K K K K% % % % %? % % %&  "  (K K K4 4 4I I I. . .D D D% % %&  >	> 	> 	>7 7 7 '""	Y'	"I.	"I.	i 	)$	I	(#	X&	  )D%=99M M :9  #"M:- - -2 2 2 [9::O O ;:O ' ' ' ' 'r   