
    Cd=z                    
   d dl mZ d dlZ ej        d          Zd dlZd dlZd dlZd dlm	Z	 d dl
mZ d dlmZmZmZ d dlmZ d dlmZ d d	lmZmZmZmZmZmZ d dlZd dlmZ d d
lmZmZmZ d dl m!Z!m"Z" d dl#m$Z$ d dl%m&Z& d dl'm(Z(m)Z) d dl*m+Z+ d dl,m-Z-m.Z. d dl/m0Z0m1Z1m2Z2 d dl3m4Z4 d e0e          v r e	ed          Z e	ed          Zej5        6                    ej7        dk    d          Z8ej5        9                    d          Z:d Z;d Z< ed          d             Z=d Z>d Z?d  Z@ej5        A                    d!ddg          d"             ZBej5        A                    d#d$d$gg          ej5        A                    d%ddg          d&                         ZCej5        A                    d'g d(          ej5        A                    d)d*d+g          d,                         ZDd- ZEd. ZFe: ed          d/                         ZGej5        jH        d0             ZIe:ej5        A                    d1d2 ejJ        d3ej5        K                    d4d5          6           ejJ        d7ej5        K                    d4d5          6          g          ej5        jH        d8                                     ZL ed          d9             ZMd: ZNd; ZOd< ZPd= ZQ ed          d>             ZRd? ZS ed          d@             ZTej5        A                    dAg dB          ej5        A                    d!g dC          dD                         ZUe:ej5        A                    dAdE ejJ        dFej5        jV        6          dG ejJ        dHej5        W                    dIJ          6          g          ej5        A                    d!g dC          ej5        A                    dKddg          dL                                                 ZXdM ZY ed          dN             ZZ ed          dO             Z[dP Z\dQ Z] ed          dR             Z^dS Z_ ed          dT             Z` ed          dU             Zaej5        A                    dVdWe.fdXe-fg           eddYgdXz  Z          d[                         Zb ed          d\             Zcd] Zdej5        9                    d           ed          d^                         Ze ed          d_             Zfd` Zgda Zhdb Zidc Zj edddeig f          dg             Zk ed          dh             ZldS )i    )annotationsNdistributed)partial)add)ClientSchedulerPluginWorkerPlugin)cleanup)client)clustercluster_fixturegen_clusterlooploop_in_threadvarying)computedelayedpersist)compute_as_if_collectionget_scheduler)	Blockwise)Delayed)
futures_ofwait)HighLevelGraph)ShuffleLayerSimpleShuffleLayer)get_named_argstmpdirtmpfile)incshould_check_stateF)r"   win32zTThe teardown of distributed.utils_test.cluster_fixture fails on windows CI currently)reasonzrignore:Running on a single-machine scheduler when a distributed client is active might lead to unexpected results.c                     ddl m}  d S )Nr   r   )dask.distributedr   r&   s    ;lib/python3.11/site-packages/dask/tests/test_distributed.pytest_can_import_clientr)   >   s    ''''''''    c                     ddl m}  d S )Nr   dumps)dask.distributed.protocolr-   r,   s    r(   test_can_import_nested_thingsr/   B   s    ////////r*   Tc                  K    t          t                    d          }t          |          \  }t          |           d {V  |j        |j        v s|j        |j        v sJ  t          t                    d          }t          |d          \  }}t          |           d {V  |j        |j        v s|j        |j        v sJ d S d S )N   
   )r   r!   r   r   keydata)	csabxx2yy2ones	            r(   test_persistr>   F   s      QAAJJER
r((NNNNNNN6QVrv////RAammGB
r((NNNNNNN6QVrv//////r*   c                d   t          d          dz   }|dz   }|dz   } t          |dd|gd| dfddg          }t          |d         d         t                    sJ t          |d         d         d         t                    sJ t          |d         d         t                    sJ d	g d
ddddgf}t	          | |k    sJ t          ||g| ddgd          }|d         d         |u sJ |d         d         |u sJ |d                                         dk    sJ |dd          ddgfk    sJ d S )Nr1         r7   r8      r   r7   r8      )r1   rA      )   rA   F)traverserF   )r   r   
isinstancer   r   )r5   r7   r8   resultsolress         r(   test_persist_nestedrL   U   sg   

QA	AA	AA1Aq!9--1vq1#>>FfQing.....fQinQ'11111fQilG,,,,,##VQ
4CFs""""
1a&!Qe
4
4
4Cq6!9>>>>q6!9>>>>q6>>q    qrr7q1#hr*   c                H   t          j        d          }t          j        d          }|                    dg di          } | j        ||g          }|                    |          }|j                            |                                |                    ||gd                     t          |j
        j        |j                 t                    sJ t          j        t                    5  |                    ddg          }d d d            d S # 1 swxY w Y   d S )	Npandasdask.dataframer9   r1   rA      r   axisr1   rA   )pytestimportorskip	DataFramescatterfrom_delayedutils	assert_eqr   concatrH   dasklayers_namer   raises	TypeError)r5   pddddffuturesddfs         r(   !test_futures_to_delayed_dataframerf   h   sC   		X	&	&B		-	.	.B	sIII&	'	'BaiR!!G
//'
"
"CHs{{}}biiRqi&A&ABBB choci0)<<<<<	y	!	! & &ooq!f%%& & & & & & & & & & & & & & & & & &s   2DDDc                T   t          j        d          }t          j        d          }|                    dt          d          i          }|                    |d          }|                    |                                          }|j                            |||            d S )NrN   rO   r9      rA   npartitions	scheduler)	rT   rU   rV   rangefrom_pandasrX   
to_delayedrY   rZ   )r5   ra   rb   rc   re   s        r(   test_from_delayed_dataframerp   y   s     
	X	&	&B		-	.	.B	sE"II&	'	'B
...
+
+C
//#..**
+
+CHsB!,,,,,r*   fusec                F   t          j        d          }t          j        d          }d}|                    t          |          t          |          d          }|                    t          |          t          |          d          }|                    ||          dz   }|                    |d          dz   }|dz  }|dz  }t
          j                            d	|i          5  |                    |d
gd          }	|		                                 |	
                                                    d
          }
d d d            n# 1 swxY w Y   |j                            |
|                    |d
gd                              d
          d           d S )NrN   rO   #   r9   r;   )r9   zri   r2   r@   optimization.fuse.activer9   left)onhowFcheck_index)rT   rU   rV   rm   rn   r\   configsetmergeheadr   sort_valuesrY   rZ   )r5   rq   ra   rb   sizedf1df2ddf1ddf2ddfmdfms              r(   $test_fused_blockwise_dataframe_merger      s   		X	&	&B		-	.	.B
 D
,,U4[[uT{{;;
<
<C
,,U4[[uT{{;;
<
<C>>#4>0025D>>#1>--2D2IC2IC	4d;	<	< . .zz$C5fz55		llnn((--. . . . . . . . . . . . . . . HSYYsu&Y11==cBBPU      s   .AEEErx   r7   	broadcastc                b   t          j        d          }t          j        d          }|                    ddgdz  t          d          d          }|                    ddgt          d          d          }|                    |d          }|                    |d          }|                    ||||d	
          }	|	                                }
|j                            |
	                    d          |                    |||          	                    d          d           d S )NrN   rO   r1   rA   rC   )r7   b_left)r7   b_rightri   tasks)rx   r   shuffler7   rx   Frz   )
rT   rU   rV   rm   rn   r~   r   rY   rZ   r   )r5   rx   r   ra   rb   pdflpdfrdfldfrr   r   s              r(   test_dataframe_broadcast_merger      s    
	X	&	&B		-	.	.B<<q!fqjE!HH==>>D<<q!fq::;;D
..1.
-
-C
..1.
-
-C88Cy'8JJD
,,..CH
t##//44      r*   computation)Nr   dask.computezscheduler, use_distributed)NT)syncFc                `  	 t          j        d          	t          j        d           G d dt                    } | j         |                       	fd} | j        |          }|                                du sJ  | j        d           }|r|dk    snd
S J |            d
S )zvShould a collection use its default scheduler or the distributed
    scheduler when being computed within a task?
    rN   rO   c                      e Zd Zd Zd ZdS )<test_default_scheduler_on_worker.<locals>.UpdateGraphCounterc                   K   d|_         d S )Nr   _update_graph_count)selfrl   s     r(   startzBtest_default_scheduler_on_worker.<locals>.UpdateGraphCounter.start   s      ,-I)))r*   c                &    |xj         dz  c_         d S Nr1   r   )r   rl   argskwargss       r(   update_graphzItest_default_scheduler_on_worker.<locals>.UpdateGraphCounter.update_graph   s    ))Q.))))r*   N)__name__
__module____qualname__r   r    r*   r(   UpdateGraphCounterr      s2        	. 	. 	.	/ 	/ 	/ 	/ 	/r*   r   c                 z   d}                      t          |           t          |           d          }                    |d          }|                               nUdk    rt	          j        |           n8dk    r0t          |j        |j        t          |j                             nJ d	S )
Nr2   rt   rA   ri   rk   r   r   FT)rV   rm   rn   r   r\   r   	__class__list)r   rc   re   r   rb   ra   rl   s      r(   fooz-test_default_scheduler_on_worker.<locals>.foo   s    \\d%++>>??nnRQn//KK)K,,,,N**L	22222666$sxch9     Ltr*   Tc                    | j         S Nr   )dask_schedulers    r(   <lambda>z2test_default_scheduler_on_worker.<locals>.<lambda>   s	    ~A r*   rA   r1   N)rT   rU   r   register_scheduler_pluginsubmitrI   run_on_scheduler)
r5   r   use_distributedrl   r   r   rK   num_update_graphsrb   ra   s
    ` `    @@r(    test_default_scheduler_on_workerr      s   * 
	X	&	&B		-	.	.B/ / / / /_ / / /  A 2 2 4 4555       " !(3--C::<<4**AA  &5N!!!!!!=N!!!!!r*   c                    g d} | j         ||g          }t          j        |          }t          |          ||z   k    sJ d S )NrP   )rW   dbrX   r   )r5   Lrd   r8   s       r(   test_futures_to_delayed_bagr      sP    		AaiAG
  A77a!er*   c                f   t          j        d          ddlm} t          j        d          }|                    d           | j        g          }                    fd|D             d          } ||                                |                    gd                     d S )N
dask.arrayr   rZ   numpyr@   c                T    g | ]$}                     |j        j                   %S ))shapedtype)rX   r   r   ).0fdar9   s     r(   
<listcomp>z1test_futures_to_delayed_array.<locals>.<listcomp>  s/    KKKa!'	9	9KKKr*   rR   )rT   rU   dask.array.utilsrZ   arangerW   concatenater   )r5   rZ   nprd   Ar   r9   s        @@r(   test_futures_to_delayed_arrayr      s    		\	*	*B******		W	%	%B
		!AaiAG
KKKKK7KKKRS 	 	 	A Iaiikk2>>1a&q>99:::::r*   c                  K   t           j                            d          5   t          t                    d                                          }d d d            n# 1 swxY w Y   t          j        d           d {V  |j        rJ  t          t                    d                              d          }t          j        d           d {V  |j        rJ d S )Nr   rk   r1   g{Gz?rA   )	r\   r|   r}   r   r!   r   asynciosleepr   r5   r6   r7   r8   r9   s        r(   &test_local_get_with_distributed_activer     s      
6	*	* & &GCLLOO##%%& & & & & & & & & & & & & & &
-

wQ&11A
-

ws   0AA#&A#c                v    t          j        d           t          j        d           ddlm}  |             d S )Nr   rN   r   )test_to_hdf)rT   rU    dask.dataframe.io.tests.test_hdfr   )r5   r   s     r(   test_to_hdf_distributedr     sE    
   
!!!<<<<<<KMMMMMr*   rj   r1   rC   zHDF not multi-process safe)r$   strict)marksr2   c                z    t          j        d           t          j        d           ddlm}  |d |            d S )Nr   rN   r   )test_to_hdf_schedulers)rT   rU   r   r   )rj   r5   r   s      r(   !test_to_hdf_scheduler_distributedr     sQ    "    
!!!GGGGGG4-----r*   c                V  K   t          j        d          }t          j        d          }|                    g dg dd          }|                    |d          }|                    dd	
                              dd          } | j        |j        d                    d {V }	 | j        |j        d                    d {V }
|j        	                    |
                    |	|
g          |                    ddgddgd                              d                     d S )NrN   rO   )r1   rA   rQ   rC   )r1   r   r1   r   rt   rA   ri   r;   F)sortcount)	split_outr   r1   )rT   rU   rV   rn   groupbyaggr   
partitionsrY   rZ   r[   	set_index)r5   r6   r7   r8   ra   rb   rc   re   rI   agg0agg1s              r(   test_serializable_groupby_aggr   7  sA     		X	&	&B		-	.	.B	LLL|||<<	=	=B
...
+
+C[[5[))--g-CCF 6,Q/00000000D6,Q/00000000DH
		4,
Aq6A//00::3??    r*   c                   t          d          t          d          }} t          t                    ||          } t          t                    ||          } t          t                    ||          } | j        |          } t          t                    |d          }|                    d          dk    sJ d S )Nr1   rA   r2   dask.distributedrk      )r   r   r   r   )r5   r9   r;   xxyyxxyyxxyy2xxyy3s           r(   test_futures_in_graphr   I  s    1::wqzzqA	a		B	a		B73<<BDAIdOOEGCLL##E==#5=66:RRRRRRRr*   c                |   t          j        d          }t          j        d           t                      5 }|                    dd          }|                    |           |                    |          }|                    |||            |j        |j        k    sJ 	 d d d            d S # 1 swxY w Y   d S )Nr   zarrrQ   rQ   r1   r1   chunksrk   )rT   rU   r   zerosto_zarr	from_zarrrZ   r   )r5   r   dr7   a2s        r(   test_zarr_distributed_roundtripr   U  s    		\	*	*B
	 %QHHVFH++			!\\!__
Qa(((yAH$$$$$% % % % % % % % % % % % % % % % % %s   A,B11B58B5c                H   t          j        d          }t          j        d          }d}|                    d|          }|                    ||          }t          j        t
          d          5  |                    |           d d d            d S # 1 swxY w Y   d S )Nr   r   r   r   r   zdistributed schedulermatch)rT   rU   ones
zeros_liker_   RuntimeErrorr   )r5   r   r   r   r7   ru   s         r(   #test_zarr_in_memory_distributed_errr  a  s    		\	*	*Bv&&DF
v&&A&))A	|+B	C	C	C  			!                 s   4BBBc                     t          d                                           |           dk    sJ  | j        fd          sJ d S )Nc                     dS r   r   r   r*   r(   r   z.test_scheduler_equals_client.<locals>.<lambda>n       r*   rk   r1   c                8    |                      j                  S r   )storyr3   )r   r9   s    r(   r   z.test_scheduler_equals_client.<locals>.<lambda>p  s    ^5I5I!%5P5P r*   )r   r   r   )r5   r9   s    @r(   test_scheduler_equals_clientr  m  sb    		A99q9!!Q&&&&1PPPPQQQQQQQr*   c                  K    t          j        t                    d          }|                                 d {V }|j        |j        v sJ |j        s	|j        sJ t          d t          |          D                       sJ d S )Nr1   c              3  >   K   | ]}|                                 V  d S r   )done)r   r   s     r(   	<genexpr>ztest_await.<locals>.<genexpr>y  s*      //Aqvvxx//////r*   )	r\   r   r!   r   r3   r   r4   allr   r   s        r(   
test_awaitr  s  s      S!AiikkA5AG6QV//A//////////r*   c                 D    d } t          j         |                        d S )Nc                    K    t          j        t                    d          } | dz   }|                                 d {V }t	          |j                   dk    sJ d S r   )r\   r   r!   r   len)r9   r;   ru   s      r(   r   ztest_local_scheduler.<locals>.f}  sf      DLa  E))++16{{ar*   )r   run)r   s    r(   test_local_schedulerr  |  s/          Kr*   c                X  K   t          j        d          }t          j        d          }ddlm} t	          t          d          t          d          ddg          fd}d	 }|                    d
d          }	t          j        d          5  |		                    ||
                    d|j                            }
d d d            n# 1 swxY w Y   t          j        d          5  |
	                    ||
                    d|j                            }d d d            n# 1 swxY w Y   t          j                            d          5   | j        |           d {V }d d d            n# 1 swxY w Y    |||                    d
          dz             d S )Nr   r   r   r   r=   tworA   c                                 | z  S r   r   )r9   scales    r(   flaky_doublez7test_annotations_blockwise_unpack.<locals>.flaky_double  s    uww{r*   c                    d| z  S )NrA   r   r9   s    r(   reliable_doublez:test_annotations_blockwise_unpack.<locals>.reliable_double  s    1ur*   r2   )r@   r   )retriesr   r   metaF)optimization__fuse__activeg      @)rT   rU   r   rZ   r   ZeroDivisionErrorr   r\   annotate
map_blocksarrayfloat_r|   r}   r   )r5   r6   r7   r8   r   r   rZ   r  r  r9   r;   ru   r  s               @r(   !test_annotations_blockwise_unpackr%    s     		\	*	*B		W	%	%B****** &u--/@/G/GANOOE       	4  A 
q	!	!	! K KLLBHHRryH,I,ILJJK K K K K K K K K K K K K K K	q	!	!	! N NLLrxx")x/L/LLMMN N N N N N N N N N N N N N N 
E	:	:  !)A,,               Ias"#####s6   2CCC12D//D36D3E==FFio)r   r   full)TFNc                   t          j        d          }t          j        d          }d}d}|dk    r-|                    ||          }|                    |          }ng|dk    r-|                    ||          }|                    |          }n4|dk    r.|                    |d	|          }|                    |d	          }|d
z  }|d
z  }t
          j                            d|i          5  |                                 t
          j	        
                    |j        |                                          }	t          |	t                    |duk    sJ |                    |||            d d d            d S # 1 swxY w Y   d S )Nr   r   )r@   rA   )r2   rC   r   r   r   r'  r2   rA   rv   Frk   )rT   rU   r   r   r'  r\   r|   r}   r   r#  optimize__dask_keys__rH   dictrZ   )
r5   r&  rq   r   r   r   r   darrnarrdsks
             r(   test_blockwise_array_creationr/    s    
	W	%	%B		\	*	*BFE	V||wwuVw,,wwu~~	wxxfx--xx	vwwubw00wwub!!AIDAID	4d;	<	< . .j!!$)T-?-?-A-ABB#t$$U):;;;;
T41---. . . . . . . . . . . . . . . . . .s   2BF  FFparquet-pyarrowzparquet-fastparquetcsvhdfr@   )rerunsfrom_futuresc                   t          j        d          }t          j        d          }|                    g ddz  t          d          d          }|rZ|j        d d         |j        dd         |j        dd         g} | j        |          }	|                    |	|d         	          }
n|                    |d
          }
|                    d          r||dk    rt          j        d           d}nt          j        d           d}|
	                    t          |          |           |                    t          |          |          }n|dk    re|
                    t          |          d           |                    t          j                            t          |          d                    }nc|dk    r]t          j        d           t           |j        d                    }|
                    |d           |                    |d          }|dg         dz   }|dg         dz   }t&          j                            d|i          5  |                                 t&          j                            |j        |                                          }t5          |t6                    t9          |          k    sJ |                    ||d           d d d            d S # 1 swxY w Y   d S )NrN   rO   rP   r@      rt   r2   r   r  rQ   ri   parquetr0  zpyarrow.parquetpyarrowfastparquet)enginer1  F)index*r2  tablesh5z/data*r9   rv   rz   )rT   rU   rV   rm   ilocrW   rX   rn   
startswith
to_parquetstrread_parquetto_csvread_csvospathjointo_hdfread_hdfr\   r|   r}   r   	dataframer)  r*  rH   r+  boolrZ   )r5   r   r&  rq   r4  ra   rb   rc   partsfutsddf0r:  re   fnr.  s                  r(   test_blockwise_dataframe_iorQ    s     
	X	&	&B		-	.	.B	IIIMb		::	;	;B 1!bgadmRWRU^<qyt%(33~~ba~00	}}Y (""" 1222FF..."FFF333ooc&kk&o99	uCKKu---kk"',,s6{{C8899	uH%%%T""##B!!!kk"h''	SERB
se*r/C	4d;	<	< 1 1n%%ch0A0A0C0CDD#t$$T

2222
S"%0001 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1s   BK77K;>K;c                p   t          j        d          }t          j        d          }|                    dg ddz  i          }|                    |d          d         }|dk     }t	          |          }|d	k    sJ |j        |                                                                         d	k    sJ d S )
NrN   rO   r9   rP   r@   rA   ri   rQ   r6  )rT   rU   rV   rn   r  r9   r   sum)r5   ra   rb   rc   seriesrI   
series_lens          r(   #test_blockwise_fusion_after_computerV    s     
	X	&	&B		-	.	.B 
sIIIM*	+	+B^^BA^..s3FaZF
 VJ4  !%%''2------r*   c           	     X  K   t          j        d          }t          j        d          fd}|                    |d|                    d          d                    d          dj                  } | j        |                                d	
           d{V }|dk    sJ dS )zDTest pack/unpack of blockwise that includes a NumPy literal argumentr   r   c                ^    t          |          j        u sJ |                     |          S r   typeuint16astyper9   dtr   s     r(   rP  z%test_blockwise_numpy_args.<locals>.fn  ,    Bxx29$$$$xx||r*   r9     *   Nr  Foptimize_graphrT   rU   	blockwiser   r[  r   rS  	r5   r6   r7   r8   r   rP  arrrK   r   s	           @r(   test_blockwise_numpy_argsrh    s       
	\	*	*B		W	%	%B     ,,
CRYYr]]D	   C 	#''))E:::
:
:
:
:
:
:C$;;;;;;r*   c           
     X  K   t          j        d          }t          j        d          dfd	}|                    |d|                    d          dj                            d                    } | j        |                                d	
           d{V }|dk    sJ dS )zLTest pack/unpack of blockwise that includes a NumPy literal keyword argumentr   r   Nc                ^    t          |          j        u sJ |                     |          S r   rY  r]  s     r(   rP  z'test_blockwise_numpy_kwargs.<locals>.fn,  r_  r*   r9   r`  ra  )r   r^  Frb  r   rd  rf  s	           @r(   test_blockwise_numpy_kwargsrk  &  s       
	\	*	*B		W	%	%B      ,,r3s")		RT,
V
VC	#''))E:::
:
:
:
:
:
:C$;;;;;;r*   c                   t          j        d          }t          j        d          }|                    |                    d                    }|                    |                    g d                    }|                                }||z  }t          j        |          \  }||z  }|                    g d          }t          j        	                    ddi          5  |
                                }	|
                                }
d d d            n# 1 swxY w Y   |j                            |	|           |j                            |
|           d S )Nr   r   rQ   )y      $@       @y      @      y       @      ?)y                y      @      @y      0@       rv   F)rT   rU   
from_arrayr   r#  conjr\   r)  r|   r}   r   testingassert_equal)r5   r   r   uvcvr9   r;   expectedx_valuey_values              r(   %test_blockwise_different_optimizationrw  5  sl   
 
	\	*	*B		W	%	%B
biill##A
bhh88899::A	
B	BAM"ER	BAxx11122H	4e<	=	=  ))++))++               JGX...JGX.....s   ')DD #D c                ^   t          j        d          }t          j        d          }|                    dd          }t          t	          |j        j                            }d||                    d          |                    d          fi}t          j	        d	||g          }|
                    |d	d
|j                  }t
          j                            ddi          5  |                    |                    d          ||            d d d            d S # 1 swxY w Y   d S )Nr   r   )d   ry  )r2   r2   r   )blockr   r   r   r1   rz  )r2   r{  r  rv   Frk   )rT   rU   r   nextiterr\   r]   int64r   from_collectionsArrayr   r|   r}   rZ   )r5   r   r   r9   namerz  r.  rg  s           r(   +test_blockwise_cull_allows_numpy_dtype_keysr  K  sN   		\	*	*B		W	%	%B 	
8,,A
 QV]##$$DtRXXa[["((1++>?E

)'51#
>
>C
((3qw(
?
?C 
4e<	=	= : :
RWWX&&q999: : : : : : : : : : : : : : : : : :s   ),D""D&)D&c                  K   t          j        d          }t          j        d          }t          j        d          }t          j        d          }d } | j        d d          }	 | j        d d	          }
|                    |d
|                    dd          d
|                    dd          d
|	ddt          |
          }|                    |                    d|	                    d	          i          d	          }|
                    dd          }|d                                         }|                                |                                z   } | j        |d           d{V }|dk    sJ dS )z9Check pack/unpack of a HLG that has every type of Layers!r   rO   r   rN   c                    | |z   |z   |z   S r   r   )r9   r;   ru   	extra_args       r(   r   z&test_combo_of_layer_types.<locals>.addj  s    1uqy9$$r*   c                    | S r   r   r  s    r(   r   z+test_combo_of_layer_types.<locals>.<lambda>m      1 r*   rA   c                    | S r   r   r  s    r(   r   z+test_combo_of_layer_types.<locals>.<lambda>n  r  r*   rQ   r9   )rQ   r1   r   NF)r   r   r  r7   ri   r   )r   rb     )rT   rU   r   re  r   r   intrn   rV   r   r   to_dask_arrayrS  r   )r5   r6   r7   r8   r   rb   r   ra   r   r;   ru   r9   rc   rK   s                 r(   test_combo_of_layer_typesr  a  s      
	\	*	*B		-	.	.B		W	%	%B		X	&	&B% % % 	a  Aa  A

d##
T""	 	 	 	A 
c299Q<<%899q	I	IB	C	)	)B	C			 	 B
%%''BFFHH
C	#e444
4
4
4
4
4
4C"999999r*   c           
        t          j        d          t          j        d          }fd}                    |                    g d                    }                    |                    g dg                    }                    |d|d|d|j        d          } | j        |d	
                               |||            dS )z1Test a blockwise operation with concatenated axesr   r   c                :                         |g dg           | S )Nr   r1   rA   r   )r9   r;   r   s     r(   r   z%test_blockwise_concatenate.<locals>.f  s"    
Q$$$r*   r  iijT)r   r   Frb  rk   N)rT   rU   rm  r#  re  r   r   rZ   )r5   r   r   r9   r;   ru   r   s         @r(   test_blockwise_concatenater    s    		\	*	*B		W	%	%B     	bhhyyy))**A
bhh			{++,,A
						g 	 		 		A AIa&&&&LLAL#####r*   c                f  K   t          j        d          }t          j        d          }|                    |                    dt	          d          i          d          } | j        |                    dd                     d {V }|d         ddd	k    sJ |d
         d
dd	k    sJ d S )NrO   rN   r7   r2   rA   ri   c                    |S r   r   )r9   partition_infos     r(   r   z4test_map_partitions_partition_info.<locals>.<lambda>  s    . r*   r   )numberdivisionr1   r@   r   )rT   rU   rn   rV   rm   r   map_partitions)r5   r6   r7   r8   rb   ra   re   rK   s           r(   "test_map_partitions_partition_infor    s      		-	.	.B		X	&	&B
..sE"II&677Q.
G
GC	HHHII       C q6q111111q6q11111111r*   c                  K   t          j        d          }t          j        d          }|                    |                    t	          t          d          |                    dddd                              d	
          }||j                            t          d                             	                                }|j
        j                                        |d<    | j        |j        j        |dgd           d{V }dS )z.Copied from distributed (tests/test_client.py)rO   rN   2   z
2020-01-01z
2020-09-01UTC)r   endperiodstz)uid
enter_timer1   ri      dayF)columnsr;  N)rT   rU   rn   rV   r+  rm   
date_ranger  isinr   r  r^  day_namer   categorical
categorize)r5   r6   r7   r8   rb   ra   re   s          r(   test_futures_in_subgraphsr    s      
	-	.	.B		X	&	&B
..
"II==&L" )    	
 	
   
 
C cgll599%%
&
.
.
0
0C"++--CJ2C%PUVVV
V
V
V
V
V
VCCCr*   zmax_branch, expected_layer_type    rA   ) r1   )r   nthreadsc                Z  K   t          j        d          }t          j        d          } G fddt                     | j                                d {V  |                    dt          d          i          }|                    |d          }	|	                    dd	|
          }
t          |
j	        j
                  t          |	j	        j
                  z
  }|D ]$}t          |
j	        j
        |         |          sJ % | j        |
           d {V  j        rJ d S )NrN   rO   c                  $    e Zd ZdZd Z fdZdS )?test_shuffle_priority.<locals>.EnsureSplitsRunImmediatelyPluginFc                    || _         d S r   )worker)r   r  s     r(   setupzEtest_shuffle_priority.<locals>.EnsureSplitsRunImmediatelyPlugin.setup  s     DKKKr*   c                   |dk    rst          d | j        j        j        D                       sMt	          d t          | j        j        j                  D                       rd_        t          d          d S d S d S )N	executingc              3  (   K   | ]}d |j         v V  dS splitNr3   r   tss     r(   r  z]test_shuffle_priority.<locals>.EnsureSplitsRunImmediatelyPlugin.transition.<locals>.<genexpr>  s:       1 1&(26!1 1 1 1 1 1r*   c              3  (   K   | ]}d |j         v V  dS r  r  r  s     r(   r  z]test_shuffle_priority.<locals>.EnsureSplitsRunImmediatelyPlugin.transition.<locals>.<genexpr>  s)      QQRw"&(QQQQQQr*   TzSplit tasks are not prioritized)	r  r  stater  anyr   readyfailurer   )r   r3   r   finishr    EnsureSplitsRunImmediatelyPlugins        r(   
transitionzJtest_shuffle_priority.<locals>.EnsureSplitsRunImmediatelyPlugin.transition  s    $$S 1 1,0K,=,G1 1 1 . .$ QQ48I8O3P3PQQQQQ J?C4<&'HIII %$$$J Jr*   N)r   r   r   r  r  r  )r  s   r(   r  r    sI        	! 	! 	!	J 	J 	J 	J 	J 	J 	Jr*   r  r7   r`  r2   ri   r   )r   
max_branch)rT   rU   r	   register_worker_pluginrV   rm   rn   r   r}   r\   r]   rH   r   r  )r5   r6   r7   r8   r  expected_layer_typera   rb   rc   re   r   shuffle_layers
layer_namer  s                @r(   test_shuffle_priorityr    sr      
	X	&	&B		-	.	.BJ J J J J J J< J J J #!
"#C#C#E#E
F
FFFFFFFF 
sE$KK(	)	)B
...
,
,C;;sG
;CCD)**S-A-AAN$ M M
$)*:68KLLLLLL
!)D///777777r*   c                  	
K   t          j        d          	t          j        d          
t          j        d          }t          j        d          }	
fd}|                    d                                          }|                    dd	
                                          } | j        |                    |||j                             d{V  dS )z7Check that map_partitions can handle a dask array inputr   rN   r   zdask.datasetsc                d    t          | j                  sJ t          |j                  sJ | S r   )rH   rV   ndarray)r   r7   r   ra   s     r(   r   z'test_map_partitions_da_input.<locals>.f  s8    !R\*****!RZ(((((r*   1d)freqr  r1   r   r  N)rT   rU   
timeseriesr   r   r   r  _meta)r5   r6   r7   r8   r   datasetsr   rc   rg  r   ra   s            @@r(   test_map_partitions_da_inputr    s       
	W	%	%B		X	&	&B		\	*	*B"?33H     
 
		$		'	'	/	/	1	1B
''$q'
!
!
)
)
+
+C
!)B%%a28%<<
=
==========r*   c            	     h   t          j        d          t          j        d          fdfd} t                              ddddidd	d	d
          5 }t                              |d          5   |              ddd           n# 1 swxY w Y   ddd           dS # 1 swxY w Y   dS )z[
    Check that map_partitions can handle a delayed
    partition of a dataframe input
    rN   rO   c                d    t          | j                  sJ t          |j                  sJ | S r   )rH   rV   )r   r7   ra   s     r(   r   z'test_map_partitions_df_input.<locals>.f  s8    !R\*****!R\*****r*   c                                                             dt          d          i          d          } |                                 d                                         }                                         dt          d          i          d          }|                    d          }|                    ||d	                                           d S )
Nr7   r2   r1   ri   r   r8   r   F)r  enforce_metadata)rn   rV   rm   ro   r   r   r  r   )item_dfre   	merged_dfrb   r   ra   s      r(   mainz*test_map_partitions_df_input.<locals>.main  s    ..sE"II.>!?!?Q.OO  ""1%--//NN2<<eBii0@#A#AqNQQ	 %%%--	  sU 	! 	
 	

')))))r*   r   z:0	dashboardFr1   )scheduler_portdashboard_addressscheduler_kwargsasynchronous	n_workersr  	processes)r  N)rT   rU   r   LocalClusterr   )r  r   rb   r   ra   s     @@@r(   test_map_partitions_df_inputr    sz   
 
	X	&	&B		-	.	.B    

 
 
 
 
 
 
 
	!	!
 %u- 
" 
 
  
e<< 	 	DFFF	 	 	 	 	 	 	 	 	 	 	 	 	 	 	                 s6   B'8BB'B	B'B	B''B+.B+c           	     L  K   t          j        d          }t          j        d          }t          j        d           |                    t          d          t          d          d          }d|j        _        |                    |d          }t                      5 }d	| }	|                    d
|	dddid          }
 | j	        |
           d {V  |j
                            ||                    d
|	d          d           d d d            d S # 1 swxY w Y   d S )NrN   rO   
sqlalchemyr2   rB   r;  r1   ri   z
sqlite:///testTechoF)r;  engine_kwargsr   )check_divisions)rT   rU   rV   rm   r;  r  rn   r    to_sqlr   rY   rZ   read_sql_table)r5   r6   r7   r8   ra   rb   rc   re   r   urirI   s              r(   test_to_sql_engine_kwargsr  -  s|      
	X	&	&B		-	.	.B
%%%	E"IIE"II66	7	7BBHM
...
+
+C	 
a1CtFE?E  
 
 ai
fc733! 	 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
s   "A*DD Dc                  
K   t          j        d          }t          j        d          } G d d          
d}|                    dg|z  t          d|dz                       }|                    |d	
          }|                    d 
fdddt                    }	 | j        |	           d {V j        dk    sJ d S )NrO   rN   c                      e Zd Zd ZdS )0test_non_recursive_df_reduce.<locals>.SomeObjectc                    || _         d S r   )val)r   r  s     r(   __init__z9test_non_recursive_df_reduce.<locals>.SomeObject.__init__Q  s    DHHHr*   N)r   r   r   r  r   r*   r(   
SomeObjectr  P  s#        	 	 	 	 	r*   r     r1   rA   )r4   r;  "   ri   c                    | S r   r   r  s    r(   r   z.test_non_recursive_df_reduce.<locals>.<lambda>X  r  r*   c                b     |                                                                             S r   )rS  )r9   r  s    r(   r   z.test_non_recursive_df_reduce.<locals>.<lambda>Y  s!    JJquuww{{}}55 r*   Fzcommit-dataset)chunk	aggregatesplit_everytokenr  )	rT   rU   Seriesrm   rn   	reductionobjectr   r  )r5   r6   r7   r8   rb   ra   NrT  dask_seriesrI   r  s             @r(   test_non_recursive_df_reducer  I  s      
	-	.	.B		X	&	&B        	AYYQC!G5AE??Y;;F..R.88K""k5555 #  F !)F########(C//////r*   c                l   t          j        d           	 t          j                            ddd                                                              dt          i          }|                    dd          }|	                                 d S # t          $ r t          j        d	           Y d S w xY w)
NrO   z
2000-01-01z
2000-07-0112h)r   r  r  	timestampT)sortedz(dd.set_index triggered a recursion error)rT   rU   r\   r  r  reset_indexr\  rB  r   r   RecursionErrorfail)r5   re   s     r(   !test_set_index_no_resursion_errorr
  b  s    
()))	@M$$<\PU$VV[]]V[#&'' 	
 mmKm55 @ @ @>??????@s   A9B B32B3c                    d} t          j        t          |           5  t          d           d d d            n# 1 swxY w Y   t          j        t          |           5  t          d           d d d            d S # 1 swxY w Y   d S )Nz	no Clientr   r   rk   r   )rT   r_   r   r   )msgs    r(   -test_get_scheduler_without_distributed_raisesr  q  s   
C	|3	/	/	/ 4 4 233334 4 4 4 4 4 4 4 4 4 4 4 4 4 4 
|3	/	/	/ / /..../ / / / / / / / / / / / / / / / / /s   ;??!A??BBc                   t                      | j        k    sJ d}t          j        t          |          5 }t          d           t          d           d d d            n# 1 swxY w Y   t          |          dk    sJ d S )NzkRunning on a single-machine scheduler when a distributed client is active might lead to unexpected results.r   threadsrk   r   rA   )r   getrT   warnsUserWarningr  )r5   warning_messageuser_warnings_as      r(   *test_get_scheduler_with_distributed_activer  z  s    ??ae####	6  
k	9	9	9 (_	****''''( ( ( ( ( ( ( ( ( ( ( ( ( ( ( 1$$$$$$s   !A$$A(+A(c                   t                      | j        k    sJ t          j                            d          5  t          j        t                    5  t                      | j        k    sJ 	 d d d            n# 1 swxY w Y   t          j                            d           5  t                      | j        k    sJ 	 d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )Nr  rk   )r   r  r\   r|   r}   rT   r  r  )r5   s    r(   7test_get_scheduler_with_distributed_active_reset_configr    s   ??ae####	9	-	- , ,\+&& 	, 	, ??ae+++++	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	,[__t_,, 	, 	, ??ae+++++	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	,, , , , , , , , , , , , , , , , , ,sY   C%A;/C%;A?	?C%A?	#C%&CC%C	C%C	C%%C),C)rl   r   )r|   r  c                   K   t          j        t                    5  t          j                                        t          j        j        k    sJ t          j        	                    d          5  t          j                                        t          j
        j        k    sJ t          | j        dd           d {V }	 t          j                                        t          j
        j        k    sJ 	 |                                 d {V  n# |                                 d {V  w xY wt          | j        dd           d {V }	 t          j                                        |j        k    sJ 	 |                                 d {V  n# |                                 d {V  w xY wt          j                                        t          j
        j        k    sJ t          | j        dd           d {V }	 t          j                                        t          j
        j        k    sJ |                                5  t          j                                        }||j        k    sJ 	 d d d            n# 1 swxY w Y   t          j                                        t          j
        j        k    sJ 	 |                                 d {V  n# |                                 d {V  w xY wt          | j        dd           d {V }t          | j        dd           d {V }	 |                                5  t          j                                        |j        k    sJ 	 d d d            n# 1 swxY w Y   t          j                                        |j        k    sJ 	 |                                 d {V  |                                 d {V  n9# |                                 d {V  |                                 d {V  w xY wt          j                                        t          j
        j        k    sJ 	 d d d            n# 1 swxY w Y   t          j                                        t          j        j        k    sJ t          | j        dd           d {V }	 t          j                                        |j        k    sJ t          j        	                    d          5  t          j                                        t          j
        j        k    sJ |                                5  t          j                                        |j        k    sJ 	 d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   |                                 d {V  n# |                                 d {V  w xY w	 d d d            d S # 1 swxY w Y   d S )Nr  rk   FT)set_as_defaultr  )rT   r  r  r\   baser   localget_syncr|   r}   threadedr  r   addressclose
as_current)r6   r   scclient2s       r(   5test_get_scheduler_default_client_config_interleavingr#    s      
k	"	" 7! 7!y&&((DJ,?????[__y_11 )	B )	B9**,,0AAAAA!!)EPTUUUUUUUUUF%y..00DM4EEEEEEllnn$$$$$$$$fllnn$$$$$$$$!!)DtTTTTTTTTTF%y..00FJ>>>>>llnn$$$$$$$$fllnn$$$$$$$$9**,,0AAAAA "!)EPTUUUUUUUUUF%y..00DM4EEEEE&&(( , ,0022B+++++, , , , , , , , , , , , , , , y..00DM4EEEEEEllnn$$$$$$$$fllnn$$$$$$$$ "!)DtTTTTTTTTTF"19UQUVVVVVVVVVG&'')) D D92244CCCCCD D D D D D D D D D D D D D Dy..00FJ>>>>>llnn$$$$$$$mmoo%%%%%%%% llnn$$$$$$$mmoo%%%%%%%%9**,,0AAAAAAS)	B )	B )	B )	B )	B )	B )	B )	B )	B )	B )	B )	B )	B )	B )	BV y&&((DJ,?????ai4PPPPPPPPP	!9**,,
::::955 C Cy..00DM4EEEEE&&(( C C92244
BBBBBC C C C C C C C C C C C C C CC C C C C C C C C C C C C C C
 ,,..        &,,..         o7! 7! 7! 7! 7! 7! 7! 7! 7! 7! 7! 7! 7! 7! 7! 7! 7! 7!sD  AW30AQ%3D6Q%D-- Q%)F8Q%F//AQ%AK
,J7KJK
J6KQ%K99=Q%7O,*N6O,NO,	N
,O,75Q%,6P""6Q%W3%Q)	)W3,Q)	-AW3A	W
AV *V	=V 	VV VV W V$	$W'V$	(W+W3W""W33W7:W7c                   K   t          j        dd          }|                    d           }t          d |j        D                       rJ d S )Nry  r2   ri   c                    | dz  S )N   r   r  s    r(   r   z*test_bag_groupby_default.<locals>.<lambda>  s
    QV r*   c              3  *   K   | ]}d |d         v V  dS )partdr   Nr   )r   ks     r(   r  z+test_bag_groupby_default.<locals>.<genexpr>  s*      44q7ad?444444r*   )r   rm   r   r  r\   )r5   r6   r7   r8   b2s        r(   test_bag_groupby_defaultr+    sZ      
"%%%A	
##	$	$B44BG4444444444r*   )m
__future__r   rT   rU   r   r   rF  sys	functoolsr   operatorr   r   r   r	   distributed.utils_testr
   r   r5   r   r   r   r   r   r   r\   dask.bagbagr   r   r   r   	dask.baser   r   dask.blockwiser   dask.delayedr   r'   r   r   dask.highlevelgraphr   dask.layersr   r   
dask.utilsr   r   r    dask.utils_testr!   markskipifplatform
pytestmarkfilterwarningsignore_sync_scheduler_warningr)   r/   r>   rL   rf   rp   parametrizer   r   r   r   r   r   xfail_with_pyarrow_stringsr   paramxfailr   r   r   r   r  r  r  r  r%  r/  skip_with_pyarrow_stringsflakyrQ  rV  rh  rk  rw  r  r  r  r  r  r  r  r  r  r  r
  r  r  r  r#  r+  r   r*   r(   <module>rF     s	   " " " " " " !f!-00  				 



             = = = = = = = = = = * * * * * * . . . . . .                       * * * * * * * * * * = = = = = = = = $ $ $ $ $ $             - - - - - - - - . . . . . . 8 8 8 8 8 8 8 8 6 6 6 6 6 6 6 6 6 6      >>+6666'+%@@@Kgg%888G [LG	(    
 !' : :2! ! ( ( (0 0 0 D0 0 0  && & &"	- 	- 	- $//  0/4 cU|,,tUm44  54 -,&       )O )O   )OX  ; ; ; D    '  (' 	+##+GPU#VV	
 	
 	
 	+##+GPU#VV	
 	
 	

  '. . ('   . D  "	S 	S 	S	% 	% 	%	 	 	R R R D0 0 0   D$ $ $<     !4!4!455. . 65 .6 !)N	
 	
 	
 	U&+"3"31"3"="=>>>  !4!4!455$77'1 '1 87 65  '1T. . .& D    D  / / /,: : :, D! ! !H$ $ $2 D	2 	2 	2 DW W W. %	 	
L  DG9q=111"8 "8 21 "8J D> > >"' ' 'T 2  D
 
 	 

. D0 0 00@ @ @/ / /	% 	% 	%, , , [&)B777:! :! 87:!z D5 5 5 5 5r*   