
    Cd	!              	         d dl mZ d dlZd dlZd dlZd dlmZ d dlmZ d dl	Z	d dl
Z
d dl
mZmZ d dlmZmZmZmZmZ d dlmZ d dlmZ d	 Zd
 Zd Zd Ze	j                            ej        dk     d          d             Zd Zd Z d Z!d Z"d Z#d Z$ G d d          Z%d Z&e	j        '                    dej(        eg          d             Z)d Z*d Z+d Z,d Z-e	j        '                    d d!d g          d"             Z. G d# d$          Z/d% Z0e	j        '                    d&d'dd(gd)z  fd*dd gd)z  fd*e0d(gd)z  fg          d+             Z1d, Z2e	j                            ej3        d-k    d.          d/             Z4e	j                            ej3        d-k    d.          d0             Z5e	j                            ej3        d-k    d1          d2             Z6e	j                            ej3        d-k    d1          d3             Z7dS )4    )annotationsN)ProcessPoolExecutor)add)computedelayed)_dumps_loadsgetget_contextremote_exception)	CPU_COUNT)incc                V    t          j        d          }|                    | g          S )Nnumpy)pytestimportorskiparrayanps     ?lib/python3.11/site-packages/dask/tests/test_multiprocessing.pyunrelated_function_globalr      s$    		W	%	%B88QC==    c                    | |z   S N r   bs     r   my_small_function_globalr      s    q5Lr   c                 R    t          t                    } d| v sJ d| vsJ d| vsJ dS )z<Unrelated globals should not be included in serialized bytes   my_small_function_globals   unrelated_function_globals   numpyN)r   r   )r   s    r   test_pickle_globalsr"      sI    '((A&!++++'q00001r   c                     t          j        d          fd} d }t          |          }d|vsJ d|v sJ d|vsJ dS )z;Unrelated locals should not be included in serialized bytesr   c                0                         | g          S r   )r   r   s    r   unrelated_function_localz4test_pickle_locals.<locals>.unrelated_function_local'   s    xx}}r   c                    | |z   S r   r   r   s     r   my_small_function_localz3test_pickle_locals.<locals>.my_small_function_local*   s    1ur   r!   s   my_small_function_locals   unrelated_function_localN)r   r   r   )r%   r'   r   r   s      @r   test_pickle_localsr(   #   s    		W	%	%B       	&''A&a////%****&a//////r      zrequires pickle protocol 5)reasonc                    t          j        d          } t          j        dd           |                     d          }g }t          ||j                  }t          |          dk    sJ t          |d         t          j                  sJ t          |d                   t          |          k    sJ t          ||	          }|                     ||k              sJ d
S )z$Test that out-of-band pickling worksr   cloudpicklez1.3.0)
minversionr)   )buffer_callback   r   )buffersN)r   r   aranger   appendlen
isinstancepicklePickleBuffer
memoryviewr	   all)r   r   lr   a2s        r   test_out_of_band_picklingr;   3   s     
	W	%	%B
'::::
		!A
Aq!(+++Aq66Q;;;;adF/00000adz!}},,,,	1			B66!r'??r   c                      t          d          )N12345
ValueErrorr   r   r   badr@   E   s    
W

r   c                     dt           fi} t          j        t                    5 }t	          | d           d d d            n# 1 swxY w Y   dt          |j                  v sJ d S )Nxr=   )r@   r   raisesr?   r
   strvalue)dskes     r   test_errors_propagaterH   I   s    -C	z	"	" aC              c!'ll""""""s   AAAc                 H   t          d          } t          | d          }t          | d          }t          |          t          |          k    sJ t          |t                     sJ dt	          |          v sJ dt	          |          v sJ dt	          |          v sJ d S )Nhelloztraceback-body	Traceback)	TypeErrorr   typer4   rD   )rG   r   r   s      r   test_remote_exceptionrN   Q   s    'A,--A,--A77d1gga#####c!ff#a&&    s1vv%%%%%%r   c                 B    dd dfd} t          | d          dk    sJ d S )N   c                    | dz   S Nr/   r   rB   s    r   <lambda>z.test_lambda_with_cloudpickle.<locals>.<lambda>^   s
    1q5 r   rB   rB   yrV      )r
   )rF   s    r   test_lambda_with_cloudpicklerX   ]   s5    #.
/
/CsC==Ar   c                     d S )Nc                    | dz   S rR   r   rS   s    r   rT   zlambda_result.<locals>.<lambda>c   s
    QU r   r   r   r   r   lambda_resultr[   b   s
    ?r   c                 \    dt           fi} t          | d          } |d          dk    sJ d S )NrB   rP   rW   )r[   r
   )rF   fs     r   $test_lambda_results_with_cloudpickler^   f   s5     
!CCA1Q441999999r   c                      e Zd Zd Zd ZdS )NotUnpickleablec                    dS )Nr   r   )selfs    r   __getstate__zNotUnpickleable.__getstate__m   s    rr   c                     t          d          )NzCan't unpickle mer>   )rb   states     r   __setstate__zNotUnpickleable.__setstate__p   s    ,---r   N)__name__
__module____qualname__rc   rf   r   r   r   r`   r`   l   s2          . . . . .r   r`   c                 V   t                      } dt          | fi}t          j        t                    5  t          |d           d d d            n# 1 swxY w Y   t          df| d}t          j        t                    5  t          |d           d d d            d S # 1 swxY w Y   d S )NrB   r   )rB   r   )r`   boolr   rC   r?   r
   )r   rF   s     r   %test_unpicklable_args_generate_errorsrl   t   s!   Aq	
C	z	"	"  C               s!
$
$C	z	"	"  C                 s#   AAA BB"%B"pool_typc                F    | t                     5 }t          j                            |          5  t	          dt
          dfid          dk    sJ t	          dt
          dfid          dk    sJ 	 d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )N)poolrB   r/   rP   )r   daskconfigsetr
   r   )rm   ro   s     r   test_reuse_poolrs      s'   	)		 2[__$_'' 	2 	2c1X,,1111c1X,,11111	2 	2 	2 	2 	2 	2 	2 	2 	2 	2 	2 	2 	2 	2 	22 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2s4   !B?A>2B>B	BB	BBBc                     t           j                            t          j        t          j                  5  t          dt          ddfdd          dk    sJ 	 d d d            d S # 1 swxY w Y   d S )N)
func_dumps
func_loadsr/   rB   rP   rU   rV   rW   )rp   rq   rr   r5   dumpsloadsr
   r   r   r   r   test_dumps_loadsry      s    	FLV\	J	J ; ;#sA//55:::::; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ;s   "A&&A*-A*c                 `    dt           dft          ddfd} t          | ddg          dk    sJ d S )Nr/   rB   
   rV   rB   rV   zr}   )rP      )r   r   r
   )ds    r   &test_fuse_doesnt_clobber_intermediatesr      s@    sCjR~66Aq3*((((((r   c                     ddl m}  dt          dft          ddfd}g  | fd	          5  t	          |d
d           d d d            n# 1 swxY w Y   t                    dk    sJ d S )Nr   )Callbackr/   rB   r{   rV   r|   c                .                         |           S r   )r2   )keyargskeyss     r   rT   z+test_optimize_graph_false.<locals>.<lambda>   s    T[[-=-= r   )pretaskr}   F)optimize_graphrP   )dask.callbacksr   r   r   r
   r3   )r   r   r   s     @r   test_optimize_graph_falser      s    ''''''sCjR~66AD	====	>	>	> * *As5))))* * * * * * * * * * * * * * *t99>>>>>>s   AAAc                      G d d          }  t          d            | d                    }t          |j        |                                          \  }t	          ||           sJ |j        dk    sJ dS )a  Previously `dask.multiprocessing.get` would accidentally forward
    `HighLevelGraph` graphs through the dask optimization/scheduling routines,
    resulting in odd errors. One way to trigger this was to have a
    non-indexable object in a task. This is just a smoketest to ensure that
    things work properly even if `HighLevelGraph` objects get passed to
    `dask.multiprocessing.get`. See https://github.com/dask/dask/issues/7190.
    c                      e Zd Zd Zd ZdS )0test_works_with_highlevel_graph.<locals>.NoIndexc                    || _         d S r   rS   )rb   rB   s     r   __init__z9test_works_with_highlevel_graph.<locals>.NoIndex.__init__   s    DFFFr   c                     t          d          )NzOh no!)	Exception)rb   r   s     r   __getitem__z<test_works_with_highlevel_graph.<locals>.NoIndex.__getitem__   s    H%%%r   N)rg   rh   ri   r   r   r   r   r   NoIndexr      s2        	 	 		& 	& 	& 	& 	&r   r   c                    | S r   r   rS   s    r   rT   z1test_works_with_highlevel_graph.<locals>.<lambda>   s    ! r   r/   N)r   r
   rp   __dask_keys__r4   rB   )r   rB   ress      r   test_works_with_highlevel_graphr      s    & & & & & & & & 	WWQZZ((A**++FSc7#####5A::::::r   randomr   c                     dk    rt          j        d          }|j         ndd l t          d           fd            d}t          j                            d          5  t          fd	t          |          D                       \  }d d d            n# 1 swxY w Y   t          t          |                    |k    sJ d S )
Nr   r   Fpurec                 T    t           fdt          d          D                       S )Nc              3  D   K   | ]}                     d d          V  dS )r   i'  N)randint).0ir   s     r   	<genexpr>z/test_random_seeds.<locals>.f.<locals>.<genexpr>   s1      @@!V^^Au--@@@@@@r   r)   )tuplerange)r   s   r   r]   ztest_random_seeds.<locals>.f   s,    @@@@uQxx@@@@@@r   r{   	processes)	schedulerc                $    g | ]}             S r   r   r   _r]   s     r   
<listcomp>z%test_random_seeds.<locals>.<listcomp>   s    444aaacc444r   )
r   r   r   r   rp   rq   rr   r   r   r3   )r   r   Nresultsr]   s   `   @r   test_random_seedsr      s(    ))%A A A A A 	A	;	/	/ 6 6444458844455
6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 s7||!!!!!!s   &+BB!$B!c                      e Zd ZdZdS )global_r   N)rg   rh   ri   rE   r   r   r   r   r      s        EEEr   r   c                     dt           _        d S rR   r   rE   r   r   r   	proc_initr      s    GMMMr   z(scheduler, initializer, expected_results	threadingr/   r{   r   c                   t          d          d             dt          _        t          j                            | |d          5  t          fdt          d          D                       \  }d d d            n# 1 swxY w Y   ||k    sJ t          fdt          d          D             | |	          \  }||k    sJ d S )
NFr   c                     t           j        S r   r   r   r   r   r]   z#test_process_initializer.<locals>.f   s
    }r   r/   )r   zmultiprocessing.initializerc                $    g | ]}             S r   r   r   s     r   r   z,test_process_initializer.<locals>.<listcomp>   s    555aaacc555r   r{   c                $    g | ]}             S r   r   r   s     r   r   z,test_process_initializer.<locals>.<listcomp>   s          r   )r   initializer)r   r   rE   rp   rq   rr   r   r   )r   r   expected_resultsr   results2r]   s        @r   test_process_initializerr      s<    %   GM	LL
 
 7 7 555559955566
7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 &&&&&    eBii     KX
 '''''''s   	+B  BBc                     ddl } d| j        v S )zhWe check for spawn by ensuring subprocess doesn't have modules only
    parent process should have:
    r   NFAKE_MODULE_FOR_TESTsysmodulesr   s    r   check_for_pytestr      s     JJJ!S[00r   win32z*Windows doesn't support different contextsc                     d } ddl }d|j        d<   	 t          j                            ddi          5  t          d| fid          }ddd           n# 1 swxY w Y   |sJ 	 |j        d= dS # |j        d= w xY w)	zThe 'multiprocessing.context' config is used to create the pool.

    We assume default is 'spawn', and therefore test for 'fork'.
    c                     dd l } d| j        v S )Nr   r   r   r   s    r   r   z@test_custom_context_used_python3_posix.<locals>.check_for_pytest   s    


%44r   r   Nr/   r   multiprocessing.contextforkrB   )r   r   rp   rq   rr   r
   )r   r   results      r   &test_custom_context_used_python3_posixr      s    5 5 5
 JJJ*+CK&'0[__7@AA 	: 	:# 023S99F	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	:K.///CK./////s.   !A. AA. AA. AA. .
A8c                    t                      t          j         d          u sJ t          j                            ddi          5  t                      t          j         d          u sJ 	 ddd           n# 1 swxY w Y   t          j                            ddi          5  t                      t          j         d          u sJ 	 ddd           dS # 1 swxY w Y   dS )zpget_context() respects configuration.

    If default context is changed this test will need to change too.
    spawnr   
forkserverNr   )r   multiprocessingrp   rq   rr   r   r   r   $test_get_context_using_python3_posixr     s|    ==O7@@@@@@	3\B	C	C J J}} ;L I IIIIIIJ J J J J J J J J J J J J J J	3V<	=	= D D}} ;F C CCCCCCD D D D D D D D D D D D D D D D D Ds$   %A88A<?A<$%CCCz!POSIX supports different contextsc                 Z   t          dt          dfid          dk    sJ t          j        t                    5  t
          j                            ddi          5  t          dt          dfid          dk    sJ 	 ddd           n# 1 swxY w Y   ddd           dS # 1 swxY w Y   dS )zOn Windows, setting 'multiprocessing.context' doesn't explode.

    Presumption is it's not used since it's unsupported, but mostly we care about
    not breaking anything.
    rB   r/   rP   r   r   N)r
   r   r   warnsUserWarningrp   rq   rr   r   r   r   %test_custom_context_ignored_elsewherer     s0    c1X$$))))	k	"	" 2 2[__7FGG 	2 	2c1X,,11111	2 	2 	2 	2 	2 	2 	2 	2 	2 	2 	2 	2 	2 	2 	22 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2s5   "B  B<B B	B B	B  B$'B$c                 :   t                      t          u sJ t          j        t                    5  t
          j                            ddi          5  t                      t          u sJ 	 ddd           n# 1 swxY w Y   ddd           dS # 1 swxY w Y   dS )z?On Python 2/Windows, get_context() always returns same context.r   r   N)r   r   r   r   r   rp   rq   rr   r   r   r   test_get_context_always_defaultr   ,  s    ==O++++	k	"	" 4 4[__7FGG 	4 	4==O33333	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	44 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4s5   "BA8,B8A<	<B?A<	 BBB)8
__future__r   r   r5   r   concurrent.futuresr   operatorr   r   rp   r   r   dask.multiprocessingr   r	   r
   r   r   dask.systemr   dask.utils_testr   r   r   r"   r(   markskipifHIGHEST_PROTOCOLr;   r@   rH   rN   rX   r[   r^   r`   rl   parametrizePoolrs   ry   r   r   r   r   r   r   r   r   platformr   r   r   r   r   r   r   <module>r      s3   " " " " " "      



 2 2 2 2 2 2         ! ! ! ! ! ! ! ! S S S S S S S S S S S S S S ! ! ! ! ! !        
    0 0 0  F+a/8TUU  VU"  # # #	& 	& 	&  
    . . . . . . . .   o&:<O%PQQ2 2 RQ2; ; ;
) ) )
    , GX#677" " 87"$          .	dQC"H%	dQC"H%	i!r* ( ( (*1 1 1 LG$P   0 0 00 LG$P   	D 	D 	D CLG+4WXX	2 	2 YX	2 CLG+4WXX4 4 YX4 4 4r   