ԋgފRddlmZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlmZmZddlmZddlmZddlmZmZmZddlmZddlmZddlZdd lmZdd l m!Z!dd l"m#Z#dd l$m%Z%dd l&m'Z'ddl(m)Z)ddl*m+Z+ddl,m-Z-ddl.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5ddl6m7Z7m8Z8ddl9m:Z:ddl;mZ>ddl?m@Z@ddlAmBZBddlCmDZDddlEmFZFddlGmHZHmIZImJZJmKZKmLZLmMZMmNZNddlOmPZPmQZQddlRmSZSmTZTmUZUejeWZXGdde<ZYGdd ZZd"d!Z[y)#) annotationsN)Callable Collection) isawaitable)Empty)ClassVarLiteralcast)merge)IOLoop) CPU_COUNT)parse_timedelta) preloading)AsyncTaskGroupClosedError)get_address_host)address_from_user_args) asyncio_run)get_loop_factory)CommClosedError ErrorMessage OKMessage RPCClosedStatuscoerce_to_address error_message) NannyPlugin_get_plugin_name)time) ServerNode) AsyncProcess)enable_proctitle_on_children)pickle) _is_dumpable)Security)get_ipget_mp_contextjson_load_robust log_errors parse_portssilence_logging_cmgrwait_for)Workerrun) DeprecatedMemoryManagerAttributeDeprecatedMemoryMonitorNannyMemoryManagerceZdZUdZej Zded<ded<ded<ded <ded <d ed <d ed < d# d$fd Ze Z e Z e Z dZd%dZedZedZfdZd&d'dZd(dZe d) d*dZed+dZ d, d-dZdZdZdZedZedZdZd.d/dZ d0 d1fd Z!d!Z"d"Z#xZ$S)2NannyaVA process to manage worker processes The nanny spins up Worker processes, watches them, and kills or restarts them as necessary. It is necessary if you want to use the ``Client.restart`` method, or to restart the worker automatically if it gets to the terminate fraction of its memory limit. The parameters for the Nanny are mostly the same as those for the Worker with exceptions listed below. Parameters ---------- env: dict, optional Environment variables set at time of Nanny initialization will be ensured to be set in the Worker process as well. This argument allows to overwrite or otherwise set environment variables for the Worker. It is also possible to set environment variables using the option ``distributed.nanny.environ``. Precedence as follows 1. Nanny arguments 2. Existing environment variables 3. Dask configuration .. note:: Some environment variables, like ``OMP_NUM_THREADS``, must be set before importing numpy to have effect. Others, like ``MALLOC_TRIM_THRESHOLD_`` (see :ref:`memtrim`), must be set before starting the Linux process. Such variables would be ineffective if set here or in ``distributed.nanny.environ``; they must be set in ``distributed.nanny.pre-spawn-environ`` so that they are set before spawning the subprocess, even if this means poisoning the process running the Nanny. For the same reason, be warned that changing ``distributed.worker.multiprocessing-method`` from ``spawn`` to ``fork`` or ``forkserver`` may inhibit some environment variables; if you do, you should set the variables yourself in the shell before you start ``dask-worker``. See Also -------- Worker z ClassVar[weakref.WeakSet[Nanny]] _instanceszWorkerProcess | Noneprocessr0memory_managerdict[str, str]env pre_spawn_env"int | str | Collection[int] | None_given_worker_port _start_portc l |tjdtdtjx|_} d|_|jttjx|_ |_ t|tr tdi|}|xs t|_t|j tsJ|j j#d|_||_|j&$t(j*j-d|_||_|j.$t(j*j-d|_|t(j*j-d}|t(j*j-d}|j0|j2|j4|j6|j8|j:|j<|j>|j@d }!tB&||!|j$|d tGjH||||jJ |_&tO||_(|r"tS||jP }"|"d|_*nk|Dt(j*j-dr%t(j*j-d|_*n%|tW||_*ntW||f|_*|.|jTjYd}#t[|#dk(r|#d}||_.|xst^|_0| |_1| |_2||_3|thn||_4tkd|_6|jlj-ddvr|jljodditq|jltkd|r-|jsD$%cic]\}$}%|$tu|%c}%}$ni|_;tqt(j*j*|xsi|_|jo||||d||_<||_=||_>| |_?| |_@|r| jt|||_Ci|_D|j|jT|_Ft|| |_H|sE|sC|jTjds(tt|jj}||_M||_N||_O||_P||_Qtjj|ycc}%}$w)Nz|the `loop` kwarg to `Nanny` is ignored, and will be removed in a future release. The Nanny always binds to the current loop. stacklevelworkerzdistributed.worker.preloadzdistributed.worker.preload-argvzdistributed.nanny.preloadzdistributed.nanny.preload-argv) instantiatekillrestartget_logs terminateclose_gracefullyr- plugin_add plugin_removeF)handlersconnection_argslocal_directory needs_workdir)file_dirtimeoutaddresszscheduler-addressz://rz#distributed.nanny.pre-spawn-environPYTHONHASHSEED)N06640zdistributed.nanny.environ)port interfaceprotocolhost)level) memory_limitz inproc://)UwarningswarnDeprecationWarning contextlib ExitStack_Nanny__exit_stackr4_setup_loggingloggerr currentloopio_loop isinstancedictr$securityget_connection_argsrJpreloaddaskconfigget preload_argvrArBrCrDcloserFr-rGrHsuper__init__rprocess_preloadsrKpreloadsr death_timeoutr'scheduler_addrrsplitlenr:r nthreads reconnectvalidate resourcesr,_get_env_variablesr8updater itemsstrr7 worker_kwargscontact_addressservicesnamequiet enter_contextr* silence_logspluginsrpc schedulerr0r5 startswithr%rrPr; _start_host _interface _protocol_listen_addressr2r3add)'self scheduler_ipscheduler_portscheduler_file worker_portrxrdrKrrrYryrzrr{rrtrjrn preload_nannypreload_nanny_argvrhrlisten_address worker_classr7rUrWrTrVrlrstackrIcfgprotocol_addresskv __class__s' 1lib/python3.12/site-packages/distributed/nanny.pyrqzNanny.__init__{snD   MM>"  %/$8$8$::E  F##)>>#33 DL h %+(+H .HJ $--222#}}@@J << ;;??+GHDL(    $ $ 0Q RD    KKOO,GHM  %!%1Q!R  ++II||  $ 5 588//!//    00+  #33 -!3d>R>R  -]; ">4;M;MNC"%i.D   !dkkoo6I&J"&++//2E"FD   #"3L"AD "3\>4R"SD   #2288? #$)+A."-%2 "  " , 4f, /0UV    ! !"2 3{ B    % %'7&@ A    : ;36399; /;41aQAY; /B  DKK.. "= #&$   +.        4< H I(/1 $"5"560LQ''22;?*4>>+A+ABCD#!- T"S 0s>T0c8d|j|jfzS)Nz)worker_addressrxrs r__repr__zNanny.__repr__&s)T-@-@$--,PPPc8K|jy|jj}|y t|jj |jdt |d{y7#t jtttf$rYywxYww)Nz nanny-close-)rP stimulus_id) r4rr+r unregisterrasyncio TimeoutErrorrOSErrorr)rrOrs r _unregisterzNanny._unregister)s <<  44  !  )) //|DF8=T*    $$ow J   s:'B?A1)A/*A1.B/A11#BBBBcJ|jdS|jjSN)r4rrs rrzNanny.worker_address:s ||+tL1L1LLrcJ|jdS|jjSr)r4 worker_dirrs rrzNanny.worker_dir>s ||+tH1H1HHrcKt |d{t|j}|D]w}t |j ||j |j|j} |j|fi|jjdd{||_ n&t!d|j d|jt#|j$|_|j(j+d{|j,j.}|j0j3|d{}d|_ |j7d|j$d d{|j9d{} |d j;D] \}} |j=| | d{"t>jAd |j$|jCd{} | tDjFk7r tId  |j7ddid{ |jOd{|jPsJ|jS|S7X7#t$r7}t|dkDr#|jtjk(rYd}~|d}~wwxYw77y7N79777#tJ$rJ |j7ddid{7n#tL$rYnwxYw|jOdd{7wxYw7#|jOd{7wxYww)z0Start nanny, start local process, start watchingN)rWrTrUrVrhr@zCould not start Nanny on host z with port zNanny->Scheduler (registration)register_nanny)oprPz nanny-plugins)pluginrz Start Nanny at: %rz$Nanny failed to start worker processstatusokerrorznanny-start-failedreason)*rp start_unsafer)r;rrrrrhlistenget_listen_args_start_addressrrwerrno EADDRINUSE ValueErrorrrPiprsstartraddrrconnectrwritereadr~rGrbinforArrunning RuntimeError Exceptionrrorstart_periodic_callbacks) rportsrT start_addressesaddrcommmsgrrresponsers rrzNanny.start_unsafeBsg"$$$D,,-D2%%// M !dkk!%)]]%B%B8%L'4#'*01A1A0BC!--.0  #4<<0mm!!#####XX%%e,,5  **$4NO O O #C 3$'$8$>$>$@LD&//d/CCC%A 8$,,G!%!1!1!33v~~-&'MNN.jj(D!1222**,  """" %%' E % u:>agg1A1A&A   $, P#D43 **h%8999'jj(P>PQQ&  #||1133  'R''  G'' jj ..7R! 4  6 Fjj(BjCCC ssEI1F$6F"7F$;I>HHH I"F$$AH8G;9HIH5I:H=;IIctKt|trtj|}t|tst j dtdtt|}| t|}|sJ||j|<tjd|t|dr( |j|}t|r |d{}t'|dd r|j)d |d  d{d diS73#t $r+}tj#d|t%|cYd}~Sd}~wwxYw7Aw)NzmRegistering duck-typed plugins has been deprecated. Please make sure your plugin inherits from `NannyPlugin`.r=r>zStarting Nanny plugin %ssetuprzNanny plugin %s failed to setuprCFz nanny-plugin-z-restartrrOK)rfbytesr"loadsrr[r\r]r rrrbrhasattrrrr exceptionrgetattrrC)rrrrrs rrGzNanny.plugin_adds fe $\\&)F&+. MML"   k6* <#F+D t# T .5 67 # (D1v&#)\F 69e ,,, dV8&D,E E E$* (  !BDI$Q'' ( FsTB$D8'"C? C= C?&D84D65D8=C?? D3 D.(D3)D8.D33D8cVKtjd| |jj|}t |dr'|j |}t |r |d{}ddiS7#t$r-}tjd|t|}|cYd}~Sd}~wwxYww)NzRemoving Nanny plugin teardownrz"Nanny plugin %s failed to teardownrr) rbrrpoprrrrrr)rrrrrrs rrHzNanny.plugin_removes ,TF34 \\%%d+Fvz*t4v&#)\F $ *    A4 H"CJ sGB)A A0%A.&A0*B).A00 B&9"B!B&B)!B&&B)cKfd} ttj||d{y7#tj$rtj d|dYyt $r}t|cYd}~Sd}~wwxYww)NcKj3jd{jd{yy77w)Nr)r4rBrA)rrsr_zNanny.restart.._sC||'iivi...&&((((.(s"AAAAAArzRestart timed out after zs; returning before finishedz timed out)r+rshieldrrbrrr)rrOrrrs` ` rrCz Nanny.restarts| ) 7>>!#.8 8 8 9##  LL*7)3OP  $ # # $sIB ';9;B ;,B'B )B1 B;B<B BB cV|jduxr|jjSrr4is_alivers rrzNanny.is_alive#||4'CDLL,A,A,CCrc"t||g|i|Sr)r-)rrargskwargss rr-z Nanny.runs4////rcp |jj|j|y#t$rYywxYwr)_ongoing_background_tasks call_soon_on_worker_exitrrexitcodes rrzNanny._on_worker_exit_syncs7   * * 4 4T5I5I8 T %   s &) 55cDK|jtjtjtjtj tj fvr |jd{ |jtjtjtj tj fvr.tjd|jd{y|jtj k(r|jdd{yy7#t$rAtjd|js|jdd{7YyYwxYw77W#t$rtj!ddYywxYww) NzFailed to unregisterznanny-unregister-failedrzRestarting workernanny-close-gracefullyz1Failed to restart worker after its process exitedTr)rrinitclosingclosedclosing_gracefullyfailedrrrbrryrowarningrArrrs rrzNanny._on_worker_exit sH ;; KK NN MM  % % MM    &&((( {{ )) # 23&&((( 9 99jj(@jAAA:!)   !78~~**,E*FFF& )A  LLCd   sAF D)0D'1D)6A2E:(E6)E:-F .2E: E8!E:%F 'D))?E3(E+)E3.F 2E33F 6E:8E:: FF FF cJ|jxr|jjSr)r4pidrs rrz Nanny.pidBs||0 0 00rcTtjdd|j|i|S)Nz'Worker._close has moved to Worker.closer=r>)r[r\ro)rrrs r_closez Nanny._closeFs' ?ANtzz4*6**rcptj|_tj d|j |y)z A signal that we shouldn't try to restart workers if they go away This is used as part of the cluster shutdown process. z*Closing Nanny gracefully at %r. Reason: %sN)rrrrbr address_safe)rrs rrFzNanny.close_gracefullyJs* //  8$:K:KV rcKjtjk(r7jd{jtjk(sJjtjk(rytj_t j dj|jjd{tjfdjDd{jjj||d{d_j j#d{tj_t$Ed{j&j)dddt j djy77777v7Mw)z; Close the worker process, stop all comms. NrzClosing Nanny at %r. Reason: %sc3@K|]}j|ywr)rH).0rrs r zNanny.close..gsQLDt11$7LsrzNanny at %r closed.)rrrfinishedrrbrrrsrrgatherrstopr4rBrrorpr`__exit__)rrOrrs` rroz Nanny.closeUsR ;;&.. (--/ ! !;;&--/ // ;;&-- 'nn  5t7H7H&Qmm$$&&&nnQDLLQRRR << #))GF); ; ; hhnnmm gmo ""4t4 )4+<+<=/ " 'R < sk1GGBG G ,G7G 86G.G /(GG*GGAGG G GGGcZK|jj||d{y7w)N)topicr)r log_eventrrrs r _log_eventzNanny._log_eventus-nn&&'   s !+)+ct|stdt|d|jj |j ||y)aLog an event under a given topic Parameters ---------- topic : str, list[str] Name of the topic under which to log an event. To log the same event under multiple topics, pass a list of topic names. msg Event message to log. Note this must be msgpack serializable. See also -------- Client.log_event z4Message must be msgpack serializable. Got type(msg)=z instead.N)r# TypeErrortyperrrrs rrzNanny.log_event{sGC GT#YL R  &&00%Mr)NNNrNNNNNautoTFFNNNNNNNNNNNNNNNNN)rr9rTr9) )z nanny-kill)rOfloatrrreturnNonerrr)rzNannyPlugin | bytesrz str | NonerErrorMessage | OKMessage)rrrr )z nanny-restart)rOrrrrz)Literal['OK', 'timed out'] | ErrorMessage)r)rrrr)rz nanny-close)rOrrrrz Literal['OK'])%__name__ __module__ __qualname____doc__weakrefWeakSetr3__annotations__rqr.rYmemory_terminate_fractionr/memory_monitorrrpropertyrrrrBrAr(rGrHrCrr-rrrrrFrorr __classcell__)rs@rr2r2Es(T4C7??3DJ0D !!&& !!;:33:;    37?d# 8 d#:1;d#N45L @ B,.NQ"MMIIEND7r>B  )  1;  !    D    2A+. 2(D0B11+  1>*- @ Nrr2ceZdZUded<ded<ded<ded<ded<d Zd Zdd Zd Zd ZdZ e dZ dZ d ddZ dZe ddZy)rz asyncio.EventrstoppedzAsyncProcess | Noner4r6r7r8g?ctj|_||_||_||_d|_||_||_||_ |j|_ ddl m }||jjdd|jjddd|_d|_y#t$rYwxYw)Nr)default_clientrshuffle)rrrrrrr4r,r7r8copyrldistributed.clientr0rrrr) rrrrr@r7r8rlr0s rrqzWorkerProcess.__init__skk (*   *kkm   9   KKOOK . KKOOIt , "    sAB11 B=<B=cKt|jtjk(r |jS|jtjk(r.|jj d{|jSt }|j|_|j|_ |jjd|jjtjj}tt!j"|j$|j&|j|j|t!j"|j(fi|j*|j,|j.dt1|_t4j.jdd|j2_|j2j9|j:t=j>|_t=j>|_ tj|_tBjDjG|jH |j2jKd{ |jW|d{} |jj[|s |jS|d |_.|d |_/|j\sJtj|_|jS77#tL$rRtNjQdd |j2jSd{7tjT|_YwxYw7#tX$r:|j2jSd{7tjT|_wxYw#|jj[wxYww) z7 Ensure the worker process is started. N)r init_result_q child_stop_quidworker_factoryr7rlz Dask Worker process (from Nanny))targetrrzdistributed.worker.daemonT)defaultzNanny failed to start processrrPdir)0r!rrrstartingwaitr&Queuer5r6putrmuuiduuid4hexr functoolspartial_runrr,rr7rlrgr4rkdaemonset_exit_callback_on_exitrEventr.osenvironr}r8rrrbrrEr_wait_until_connectedrsetrr)rmp_ctxr7rs rrzWorkerProcess.starts %& ;;&.. (;;  ;;&// ),,##% % %;; !#\\^"LLN d# jjl#$$ !.."00!..(00S@R@RSHH{{ 46   #kkoo.ISWoX  &&t}}5}} }} oo  $,,-  ,ll((***  66s;; LL   ;; !)ne*""""nn {{A &P+ ,  !@4 Pll,,...$mm  ,< ll,,...$mm    LL   sA.O0L1G6O)LLL M6 M4!M6&A,OL=M1MM1.N<0M11N<4M66&N9NN99N<<OOcB||jury|jyr)r4 mark_stopped)rprocs rrHzWorkerProcess._on_exits t|| #  rcH|J|dk(rd|fzS|dk\rd||fzSd|| fzS)Nz.Worker process %d was killed by unknown signalrz'Worker process %d exited with status %dz)Worker process %d was killed by signal %drZ)rrrs r_death_messagezWorkerProcess._death_messagesJ### s?CsfL L ]<XN N>#yAQQ QrcV|jduxr|jjSrrrs rrzWorkerProcess.is_aliverrc~|jr0|jjr|jjSdSr)r4rrrs rrzWorkerProcess.pids-#'< C  ..DK LL   LL    LL   !%D  $D DL277>>$//#B dooTB"DO||' Q() )rcKt|z}|jtjk(r"|jj d{|jtj tjtjtjtjfvsJ|j|jtjk(ry|jtj k(r#|jj d{ytj |_tjd||j}|dz}|jjd|||d|jj|J |j!|d{y7c77 #t"j$$rYnwxYwtj'd|dd|j)d{7|j!t+d |tz d{7y#t,$r}d t/|vrYd}~yd}~wwxYww) aX Ensure the worker process is stopped, waiting at most ``timeout * 0.8`` seconds before killing it abruptly. When `kill` returns, the worker process has been joined. If the worker process does not terminate within ``timeout`` seconds, even after being killed, `asyncio.TimeoutError` is raised. Nz(Nanny asking worker to close. Reason: %sg?r)rrO executor_waitrz!Worker process still alive after z.1fz seconds, killingrz(invalid operation on closed AsyncProcess)rrrr<rr=stoppingr.rrrbrr4r6r?rojoinrrrrBmaxrr)rrOr`rdeadliner4 wait_timeoutrs rrBzWorkerProcess.kill6s6G# ;;&// ),,##% % %{{ OO NN NN MM  % %     ;;    ;;&.. (  ;;&// ),,##% % % oo  >G,,}  '!.    !"""  ll<000A & &&1''   NN3L3EEVW ,,. ,,s1h&789 9 9 9SVC  sAI F- CIF0A?IF4'F2(F4,I0I2F44G H0 G  0H0:G=;.H0)H,*H0/I0 I9 I I I  IIcK |jtjk7ry |jj }|d|k7rCd|vr|d|S#t $r+t j|jd{7YwxYww)Nr7r) rrr<r5 get_nowaitrrsleep_init_msg_interval)rr7rs rrLz#WorkerProcess._wait_until_connectedys{{foo- ((335 5zS c!+&&  mmD$;$;<<< s3 BAB+B<A?=BBBBc dddd d d d  fd d  fd }tj5 jdfd } tjj |t jjt jj|d d l m } | |rtj|t|t dddy#1swYyxYw)NrTzworkerprocess-stop)rOr`rcJK|jd|||d{y7w)NF)rr`rOr)ro)r@rOr`rs rdo_stopz#WorkerProcess._run..do_stops/,,+   s #!#ct j}j |ddk(sJ||d=|jfd|i|y#tttf$rtj dddi}YgwxYw#j ddk(sJ||d=|jfd|i|wxYw)zi Wait for an incoming stop message and then stop the worker cleanly. z Worker process died unexpectedlyrrr@N)rmrrEOFErrorrbrro add_callback)rdr@rr6rls r watch_stop_qz(WorkerProcess._run..watch_stop_qs A"&&( ""$4yF*/C/*I!!!'@&@C@w1 % ?@Vn %""$4yF*/C/*I!!!'@&@C@s#A -A;8A>:A;;A>>9B7cpKd} }d}tjtj|t j dd}|j j|jd|4d {d } |jsJ j|j|jd j|jd {tjd d d d d {j'y 77<#t $rY0wxYw7(#1d {7swY8xYw#t"$rJ}|tj%d |d j|djYd }~d }~wwxYw#j'wxYww)zK Try to start worker and inform parent of outcome. initializer)r@rdzNanny stop queue watchT)r9rrFr=rNN)rPr;r7z Worker closedz Failed to z worker)r7r) threadingThreadrCrDr rcrcallbackrbrPr?rKror rbrrrr join_thread) failure_typer@threadrr5rr7rpr8s rr-zWorkerProcess._run..runs(4L( ,')& "))$,,$%#^^- 2 v{{A6!6#'L5%~~-~&))+1>>'-'='='*&++-$oo/// O4#"64))+5" 0& "666$ &'  :l^7!CD!!#A">?##%%  &))+sF6A.close_stop_qs` $$dF^4 &&(((*" "s 8A AA AAr)default_initializer) loop_factory) r@r,rOrr`boolrrrr)rdr r@r,rr)rr)r^r_rurJrKr}rkrlrefreshrMdask.multiprocessingr{rbsetLevelrr)clsrr5r6r7r7rlr8r-rzr{rlrrps ``` ` @@@rrEzWorkerProcess._runs"&.         A - ,- ,^ ! ! #u ^^ + + JJ  c " KK   ! KKOOF # @  ! - ,<,> ?5$ # #s B0C88DNr)rTzworkerprocess-kill)rOrr`r}rrrr)rr}r5multiprocessing.Queuer6rr7rr7rgrlrgr8zCallable[[], Worker]rr)r"r#r$r(rirqrrHrTrr+rrPrBrL classmethodrErZrrrrs    !! #DHT RDVV 4"* AAA A  AF(q@q@-q@, q@  q@  q@q@-q@ q@q@rrc <tjj|}t|tst |dt ||jDcic]2\}}| |tjj|t|4}}}|Scc}}w)Nz1 configuration must be of type dict. Instead got ) rkrlrmrfrgrrr~rJrKr) config_keyrrrs rr|r|s ++//* %C c4 lKDQTI; W  58IIK QKDAq1=1bjjnnQA' 'KC Q J Rs  B',B)rrrr6)\ __future__rrr^rrCloggingmultiprocessingrJr\rsr@r[r&collections.abcrrinspectrqueuertypingrr r toolzr tornado.ioloopr rk dask.systemr dask.utilsr distributedrdistributed._async_taskgrouprdistributed.commrdistributed.comm.addressingrdistributed.compatibilityrdistributed.configrdistributed.corerrrrrrrdistributed.diagnostics.pluginrrdistributed.metricsrdistributed.noderdistributed.processr distributed.proctitler!distributed.protocolr"distributed.protocol.serializer#distributed.securityr$distributed.utilsr%r&r'r(r)r*r+distributed.workerr,r-distributed.worker_memoryr.r/r0 getLoggerr"rbr2rr|rZrrrs"   0**! !&"B->1/I$',>'7)+   8 $I NJI NXn@n@b r