
    ∋dn                    d   d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlZd dlmZ d dlmZmZmZ d dlmZmZ erd dlmZmZ  ej        e          Z G d d          Z G d	 d
          Z G d d          Zd(dZ G d de          Z G d deej                   Z! G d de!          Z" G d de!          Z# G d de          Z$ G d de          Z% G d dej&                  Z' G d  d!e          Z( G d" d#e          Z) G d$ d%          Z* G d& d'e          Z+dS ))    )annotationsN)	Awaitable)TYPE_CHECKINGAnyClassVar)funcnametmpfile)	SchedulerTaskStateStatec                  j    e Zd ZdZd)dZd*dZd*dZd+dZd)dZd,dZ	d-d!Z
d-d"Zd.d#Zd.d$Zd/d'Zd(S )0SchedulerPlugina  Interface to extend the Scheduler

    A plugin enables custom hooks to run when specific events occur.  The    scheduler will run the methods of this plugin whenever the corresponding
    method of the scheduler is run.  This runs user code within the scheduler
    thread that can perform arbitrary operations in synchrony with the scheduler
    itself.

    Plugins are often used for diagnostics and measurement, but have full
    access to the scheduler and could in principle affect core scheduling.

    To implement a plugin:

    1. subclass this class
    2. override some of its methods
    3. add the plugin to the scheduler with ``Scheduler.add_plugin(myplugin)``.

    Examples
    --------
    >>> class Counter(SchedulerPlugin):
    ...     def __init__(self):
    ...         self.counter = 0
    ...
    ...     def transition(self, key, start, finish, *args, **kwargs):
    ...         if start == 'processing' and finish == 'memory':
    ...             self.counter += 1
    ...
    ...     def restart(self, scheduler):
    ...         self.counter = 0

    >>> plugin = Counter()
    >>> scheduler.add_plugin(plugin)  # doctest: +SKIP
    	schedulerr
   returnNonec                
   K   dS )zhRun when the scheduler starts up

        This runs at the end of the Scheduler startup process
        N selfr   s     >lib/python3.11/site-packages/distributed/diagnostics/plugin.pystartzSchedulerPlugin.start:   
            c                
   K   dS )z*Runs prior to any Scheduler shutdown logicNr   r   s    r   before_closezSchedulerPlugin.before_close@   r   r   c                
   K   dS )zRun when the scheduler closes down

        This runs at the beginning of the Scheduler shutdown process, but after
        workers have been asked to shut down gracefully
        Nr   r   s    r   closezSchedulerPlugin.closeC   r   r   clientstrkeysset[str]tasks	list[str]r   dict[str, dict[str, Any]]priority"dict[str, tuple[int | float, ...]]dependenciesdict[str, set]kwargsr   c                   dS )a  Run when a new graph / tasks enter the scheduler

        Parameters
        ----------
            scheduler:
                The `Scheduler` instance.
            client:
                The unique Client id.
            keys:
                The keys the Client is interested in when calling `update_graph`.
            tasks:
                The
            annotations:
                Fully resolved annotations as applied to the tasks in the format::

                    {
                        "annotation": {
                            "key": "value,
                            ...
                        },
                        ...
                    }
            priority:
                Task calculated priorities as assigned to the tasks.
            dependencies:
                A mapping that maps a key to its dependencies.
            **kwargs:
                It is recommended to allow plugins to accept more parameters to
                ensure future compatibility.
        Nr   )	r   r   r   r    r"   r   r%   r'   r)   s	            r   update_graphzSchedulerPlugin.update_graphJ         r   c                    dS )z&Run when the scheduler restarts itselfNr   r   s     r   restartzSchedulerPlugin.restartu   r,   r   keyr   r   finishargsc                    dS )a  Run whenever a task changes state

        For a description of the transition mechanism and the available states,
        see :ref:`Scheduler task states <scheduler-task-state>`.

        .. warning::

            This is an advanced feature and the transition mechanism and details
            of task states are subject to change without deprecation cycle.

        Parameters
        ----------
        key : string
        start : string
            Start state of the transition.
            One of released, waiting, processing, memory, error.
        finish : string
            Final state of the transition.
        *args, **kwargs :
            More options passed when transitioning
            This may include worker ID, compute time, etc.
        Nr   )r   r/   r   r0   r1   r)   s         r   
transitionzSchedulerPlugin.transitionx   r,   r   workerNone | Awaitable[None]c                    dS )a>  Run when a new worker enters the cluster

        If this method is synchronous, it is immediately and synchronously executed
        without ``Scheduler.add_worker`` ever yielding to the event loop.
        If it is asynchronous, it will be awaited after all synchronous
        ``SchedulerPlugin.add_worker`` hooks have executed.

        .. warning::

            There are no guarantees about the execution order between individual
            ``SchedulerPlugin.add_worker`` hooks and the ordering may be subject
            to change without deprecation cycle.
        Nr   r   r   r4   s      r   
add_workerzSchedulerPlugin.add_worker   r,   r   c                    dS )aC  Run when a worker leaves the cluster

        If this method is synchronous, it is immediately and synchronously executed
        without ``Scheduler.remove_worker`` ever yielding to the event loop.
        If it is asynchronous, it will be awaited after all synchronous
        ``SchedulerPlugin.remove_worker`` hooks have executed.

        .. warning::

            There are no guarantees about the execution order between individual
            ``SchedulerPlugin.remove_worker`` hooks and the ordering may be subject
            to change without deprecation cycle.
        Nr   r7   s      r   remove_workerzSchedulerPlugin.remove_worker   r,   r   c                    dS )zRun when a new client connectsNr   r   r   r   s      r   
add_clientzSchedulerPlugin.add_client   r,   r   c                    dS )zRun when a client disconnectsNr   r<   s      r   remove_clientzSchedulerPlugin.remove_client   r,   r   topicmsgc                    dS )zRun when an event is loggedNr   )r   r@   rA   s      r   	log_eventzSchedulerPlugin.log_event   r,   r   Nr   r
   r   r   r   r   )r   r
   r   r   r    r!   r"   r#   r   r$   r%   r&   r'   r(   r)   r   r   r   )r/   r   r   r   r0   r   r1   r   r)   r   r   r   )r   r
   r4   r   r   r5   )r   r
   r   r   r   r   )r@   r   rA   r   r   r   )__name__
__module____qualname____doc__r   r   r   r+   r.   r3   r8   r:   r=   r?   rC   r   r   r   r   r      s         B   9 9 9 9   ) ) ) )V5 5 5 5   >      "- - - -, , , ,* * * * * *r   r   c                  $    e Zd ZdZd Zd Zd ZdS )WorkerPluginaW  Interface to extend the Worker

    A worker plugin enables custom code to run at different stages of the Workers'
    lifecycle.

    A plugin enables custom code to run at each of step of a Workers's life. Whenever such
    an event happens, the corresponding method on this class will be called. Note that the
    user code always runs within the Worker's main thread.

    To implement a plugin implement some of the methods of this class and register
    the plugin to your client in order to have it attached to every existing and
    future workers with ``Client.register_worker_plugin``.

    Examples
    --------
    >>> class ErrorLogger(WorkerPlugin):
    ...     def __init__(self, logger):
    ...         self.logger = logger
    ...
    ...     def setup(self, worker):
    ...         self.worker = worker
    ...
    ...     def transition(self, key, start, finish, *args, **kwargs):
    ...         if finish == 'error':
    ...             ts = self.worker.tasks[key]
    ...             exc_info = (type(ts.exception), ts.exception, ts.traceback)
    ...             self.logger.error(
    ...                 "Error during computation of '%s'.", key,
    ...                 exc_info=exc_info
    ...             )

    >>> import logging
    >>> plugin = ErrorLogger(logging)
    >>> client.register_worker_plugin(plugin)  # doctest: +SKIP
    c                    dS )z
        Run when the plugin is attached to a worker. This happens when the plugin is registered
        and attached to existing workers, or when a worker is created after the plugin has been
        registered.
        Nr   r   r4   s     r   setupzWorkerPlugin.setup   r,   r   c                    dS )zeRun when the worker to which the plugin is attached is closed, or
        when the plugin is removed.Nr   rM   s     r   teardownzWorkerPlugin.teardown   r,   r   c                    dS )ar  
        Throughout the lifecycle of a task (see :doc:`Worker State
        <worker-state>`), Workers are instructed by the scheduler to compute
        certain tasks, resulting in transitions in the state of each task. The
        Worker owning the task is then notified of this state transition.

        Whenever a task changes its state, this method will be called.

        .. warning::

            This is an advanced feature and the transition mechanism and details
            of task states are subject to change without deprecation cycle.

        Parameters
        ----------
        key : string
        start : string
            Start state of the transition.
            One of waiting, ready, executing, long-running, memory, error.
        finish : string
            Final state of the transition.
        kwargs : More options passed when transitioning
        Nr   )r   r/   r   r0   r)   s        r   r3   zWorkerPlugin.transition   r,   r   N)rF   rG   rH   rI   rN   rP   r3   r   r   r   rK   rK      sL        " "H  ' ' '    r   rK   c                  "    e Zd ZdZdZd Zd ZdS )NannyPlugina  Interface to extend the Nanny

    A worker plugin enables custom code to run at different stages of the Workers'
    lifecycle. A nanny plugin does the same thing, but benefits from being able
    to run code before the worker is started, or to restart the worker if
    necessary.

    To implement a plugin implement some of the methods of this class and register
    the plugin to your client in order to have it attached to every existing and
    future nanny by passing ``nanny=True`` to
    :meth:`Client.register_worker_plugin<distributed.Client.register_worker_plugin>`.

    The ``restart`` attribute is used to control whether or not a running ``Worker``
    needs to be restarted when registering the plugin.

    See Also
    --------
    WorkerPlugin
    SchedulerPlugin
    Fc                    dS )z
        Run when the plugin is attached to a nanny. This happens when the plugin is registered
        and attached to existing nannies, or when a nanny is created after the plugin has been
        registered.
        Nr   r   nannys     r   rN   zNannyPlugin.setup#  r,   r   c                    dS )z?Run when the nanny to which the plugin is attached to is closedNr   rU   s     r   rP   zNannyPlugin.teardown*  r,   r   N)rF   rG   rH   rI   r.   rN   rP   r   r   r   rS   rS     sH         * G  N N N N Nr   rS   plugin,SchedulerPlugin | WorkerPlugin | NannyPluginr   r   c                    t          | d          r| j        S t          t          |                     dz   t	          t          j                              z   S )zUReturn plugin name.

    If plugin has no name attribute a random name is used.

    name-)hasattrr[   r   typer   uuiduuid4)rX   s    r   _get_plugin_namera   .  sK     vv @{V%%+c$*,,.?.???r   c                  $    e Zd ZdZdddZddZdS )SchedulerUploadFileupload_fileTfilepathr   loadboolc                    t           j                            |          | _        || _        t          |d          5 }|                                | _        ddd           dS # 1 swxY w Y   dS S
        Initialize the plugin by reading in the data from the given file.
        rbNospathbasenamefilenamerf   openreaddatar   re   rf   fs       r   __init__zSchedulerUploadFile.__init__=       ((22	(D!! 	!QDI	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	!   A##A'*A'r   r
   r   r   c                d   K   |                     | j        | j        | j                   d {V  d S )N)rf   )rd   rp   rs   rf   r   s     r   r   zSchedulerUploadFile.startF  s?      ##DM4949#MMMMMMMMMMMr   NTre   r   rf   rg   rD   )rF   rG   rH   r[   rv   r   r   r   r   rc   rc   :  sL        D! ! ! ! !N N N N N Nr   rc   c                      e Zd ZU dZded<   ded<   ded<   ded	<   dd
Zd Zej        dd            Z	d Z
d Zd Zd Zd Zd ZdS )PackageInstalla1  Abstract parent class for a worker plugin to install a set of packages

    This accepts a set of packages to install on all workers.
    You can also optionally ask for the worker to restart itself after
    performing this installation.

    .. note::

       This will increase the time it takes to start up
       each worker. If possible, we recommend including the
       libraries in the worker environment or image. This is
       primarily intended for experimentation and debugging.

    Parameters
    ----------
    packages
        A list of packages (with optional versions) to install
    restart
        Whether or not to restart the worker after installing the packages
        Only functions if the worker has an attached nanny process

    See Also
    --------
    CondaInstall
    PipInstall
    zClassVar[str]	INSTALLERr   r[   r#   packagesrg   r.   c                f    || _         || _        | j         dt          j                     | _        d S )Nz	-install-)r   r.   r~   r_   r`   r[   )r   r   r.   s      r   rv   zPackageInstall.__init__l  s3    
 !~>>
>>			r   c                   K   ddl m}  |dt          j                    d|j        |j                   d {V 4 d {V  |                     |           d {V sVt                              d| j	        | j
                   |                     |           d {V  |                                  n t                              d| j
                   | j        r|j        rz|                     |           d {V s_t                              d           |                     |           d {V  |j                            |j        d| j         d	
           d d d           d {V  d S # 1 d {V swxY w Y   d S )Nr   )	Semaphore   T)
max_leasesr[   registerscheduler_rpcloopz(%s installing the following packages: %sz6The following packages have already been installed: %sz)Restarting worker to refresh interpreter.z-setup)r.   reason)distributed.semaphorer   socketgethostnamer   r   _is_installedloggerinfor~   r   _set_installedinstallr.   rV   _is_restarted_set_restartedadd_callbackclose_gracefullyr[   )r   r4   r   s      r   rN   zPackageInstall.setupu  su     333333 )'))$.[        	 	 	 	 	 	 	 	 ++F33333333 >NM  
 ))&111111111LM  
 |  4;M;Mf;U;U5U5U5U5U5U5U GHHH))&111111111((+TTYBVBVBV )   5	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	s    DE--
E7:E7r   r   c                    dS )zInstall the requested packagesNr   r   s    r   r   zPackageInstall.install  r,   r   c                n   K   |j                             |                                 d           d {V S NF)default)r   get_metadata_compose_installed_keyrM   s     r   r   zPackageInstall._is_installed  sT      ]//''))5 0 
 
 
 
 
 
 
 
 	
r   c                p   K   |j                             |                                 d           d {V  d S NT)r   set_metadatar   rM   s     r   r   zPackageInstall._set_installed  sZ      m((''))
 
 	
 	
 	
 	
 	
 	
 	
 	
 	
r   c                8    | j         dt          j                    gS )N	installed)r[   r   r   r   s    r   r   z%PackageInstall._compose_installed_key  s     I  
 	
r   c                p   K   |j                             |                     |          d           d {V S r   )r   r   _compose_restarted_keyrM   s     r   r   zPackageInstall._is_restarted  sW      ]//''// 0 
 
 
 
 
 
 
 
 	
r   c                r   K   |j                             |                     |          d           d {V  d S r   )r   r   r   rM   s     r   r   zPackageInstall._set_restarted  s\      m((''//
 
 	
 	
 	
 	
 	
 	
 	
 	
 	
r   c                     | j         d|j        gS )N	restarted)r[   rV   rM   s     r   r   z%PackageInstall._compose_restarted_key  s    	;55r   N)r   r#   r.   rg   rE   )rF   rG   rH   rI   __annotations__rv   rN   abcabstractmethodr   r   r   r   r   r   r   r   r   r   r}   r}   J  s          6 IIIMMM? ? ? ?  B 	- - - -
 
 


 
 

 
 

 
 

 
 
6 6 6 6 6r   r}   c                  B     e Zd ZU dZdZded<   	 	 dd fdZddZ xZS )CondaInstallam  A Worker Plugin to conda install a set of packages

    This accepts a set of packages to install on all workers as well as
    options to use when installing.
    You can also optionally ask for the worker to restart itself after
    performing this installation.

    .. note::

       This will increase the time it takes to start up
       each worker. If possible, we recommend including the
       libraries in the worker environment or image. This is
       primarily intended for experimentation and debugging.

    Parameters
    ----------
    packages
        A list of packages (with optional versions) to install using conda
    conda_options
        Additional options to pass to conda
    restart
        Whether or not to restart the worker after installing the packages
        Only functions if the worker has an attached nanny process

    Examples
    --------
    >>> from dask.distributed import CondaInstall
    >>> plugin = CondaInstall(packages=["scikit-learn"], conda_options=["--update-deps"])

    >>> client.register_worker_plugin(plugin)

    See Also
    --------
    PackageInstall
    PipInstall
    condar#   conda_optionsNFr   list[str] | Noner.   rg   c                `    t                                          ||           |pg | _        d S N)r.   )superrv   r   )r   r   r   r.   	__class__s       r   rv   zCondaInstall.__init__  s5     	7333*0br   r   r   c                   	 ddl m}m} n># t          $ r1}d}t                              |           t          |          |d }~ww xY w	  ||j        | j        | j	        z             \  }}}n># t          $ r1}d}t                              |           t          |          |d }~ww xY w|dk    rSd|                                                                 d}t                              |           t          |          d S )Nr   )Commandsrun_commandz`conda install failed because conda could not be found. Please make sure that conda is installed.zconda install failedzconda install failed with '')conda.cli.python_apir   r   ModuleNotFoundErrorr   errorRuntimeErrorINSTALLr   r   	Exceptiondecodestrip)r   r   r   erA   _stderr
returncodes           r   r   zCondaInstall.install  s>   	+BBBBBBBBB" 	+ 	+ 	+<  LLs##*	+	+$/K $"4t}"D% %!Avzz  	+ 	+ 	+(CLLs##*	+
 ??J0E0E0G0GJJJCLLs### ?s,    
A,AA
"A- -
B(7,B##B(NF)r   r#   r   r   r.   rg   rE   	rF   rG   rH   rI   r~   r   rv   r   __classcell__r   s   @r   r   r     s~         # #J I
 +/	1 1 1 1 1 1 1$ $ $ $ $ $ $ $r   r   c                  B     e Zd ZU dZdZded<   	 	 dd fdZddZ xZS )
PipInstalla]  A Worker Plugin to pip install a set of packages

    This accepts a set of packages to install on all workers as well as
    options to use when installing.
    You can also optionally ask for the worker to restart itself after
    performing this installation.

    .. note::

       This will increase the time it takes to start up
       each worker. If possible, we recommend including the
       libraries in the worker environment or image. This is
       primarily intended for experimentation and debugging.

    Parameters
    ----------
    packages
        A list of packages (with optional versions) to install using pip
    pip_options
        Additional options to pass to pip
    restart
        Whether or not to restart the worker after installing the packages
        Only functions if the worker has an attached nanny process

    Examples
    --------
    >>> from dask.distributed import PipInstall
    >>> plugin = PipInstall(packages=["scikit-learn"], pip_options=["--upgrade"])

    >>> client.register_worker_plugin(plugin)

    See Also
    --------
    PackageInstall
    CondaInstall
    pipr#   pip_optionsNFr   r   r.   rg   c                `    t                                          ||           |pg | _        d S r   )r   rv   r   )r   r   r   r.   r   s       r   rv   zPipInstall.__init__2  s5     	7333&,"r   r   r   c                   t          j        t          j        dddg| j        z   | j        z   t           j        t           j                  }|                                \  }}|                                }|dk    rSd|	                                
                                 d}t                              |           t          |          d S )Nz-mr   r   )stdoutr   r   zpip install failed with 'r   )
subprocessPopensys
executabler   r   PIPEcommunicatewaitr   r   r   r   r   )r   procr   r   r   rA   s         r   r   zPipInstall.install;  s    ^T5)4t7GG$-W??
 
 

 $$&&	6YY[[
??Hfmmoo.C.C.E.EHHHCLLs### ?r   r   )r   r#   r   r   r.   rg   rE   r   r   s   @r   r   r     s~         # #J I
 )-	- - - - - - -$ $ $ $ $ $ $ $r   r   c                  &    e Zd ZdZdZdddZd	 Zd
S )
UploadFileaQ  A WorkerPlugin to upload a local file to workers.

    Parameters
    ----------
    filepath: str
        A path to the file (.py, egg, or zip) to upload

    Examples
    --------
    >>> from distributed.diagnostics.plugin import UploadFile

    >>> client.register_worker_plugin(UploadFile("/path/to/file.py"))  # doctest: +SKIP
    rd   Tre   r   rf   rg   c                    t           j                            |          | _        || _        t          |d          5 }|                                | _        ddd           dS # 1 swxY w Y   dS ri   rl   rt   s       r   rv   zUploadFile.__init__[  rw   rx   c                   K   |                     | j        | j        | j                   d {V }t	          | j                  |d         k    sJ d S )N)rp   rs   rf   nbytes)rd   rp   rs   rf   len)r   r4   responses      r   rN   zUploadFile.setupd  sq      ++] , 
 
 
 
 
 
 
 
 49~~(!3333333r   Nrz   r{   )rF   rG   rH   rI   r[   rv   rN   r   r   r   r   r   J  sM          D! ! ! ! !4 4 4 4 4r   r   c                  $    e Zd ZdZd Zd Zd ZdS )ForwardLoggingPlugina  
    A ``WorkerPlugin`` to forward python logging records from worker to client.
    See :meth:`Client.forward_logging` for full documentation and usage. Needs
    to be used in coordination with :meth:`Client.subscribe_topic`, the details
    of which :meth:`Client.forward_logging` handles for you.

    Parameters
    ----------
    logger_name : str
        The name of the logger to begin forwarding.

    level : str | int
        Optionally restrict forwarding to ``LogRecord``s of this level or
        higher, even if the forwarded logger's own level is lower.

    topic : str
        The name of the topic to which to the worker should log the forwarded log
        records.
    c                >    || _         || _        || _        d | _        d S N)logger_namelevelr@   handler)r   r   r   r@   s       r   rv   zForwardLoggingPlugin.__init__  s#    &

r   c                    t          || j        | j                  | _        t	          j        | j                  }|                    | j                   d S )N)r   )_ForwardingLogHandlerr@   r   r   logging	getLoggerr   
addHandlerr   r4   r   s      r   rN   zForwardLoggingPlugin.setup  sK    ,VTZtzRRR"4#344$,'''''r   c                ~    | j         5t          j        | j                  }|                    | j                    d S d S r   )r   r   r   r   removeHandlerr   s      r   rP   zForwardLoggingPlugin.teardown  sA    <#&t'788F  ..... $#r   N)rF   rG   rH   rI   rv   rN   rP   r   r   r   r   r   k  sK         (  ( ( (
/ / / / /r   r   c                  <     e Zd ZdZej        f fd	Zd Zd Z xZ	S )r   a*  
    Handler class that gets installed inside workers by
    :class:`ForwardLoggingPlugin`. Not intended to be instantiated by the user
    directly.

    In each affected worker, ``ForwardLoggingPlugin`` adds an instance of this
    handler to one or more loggers (possibly the root logger). Tasks running on
    the worker may then use the affected logger as normal, with the side effect
    that any ``LogRecord``s handled by the logger (or by a logger below it in
    the hierarchy) will be published to the dask client as a
    ``topic`` event.
    c                f    t                                          |           || _        || _        d S r   )r   rv   r4   r@   )r   r4   r@   r   r   s       r   rv   z_ForwardingLogHandler.__init__  s-    


r   c                    |j         }|r|                     |          }t          |j                  }|                                |d<   d |d<   d |d<   |                    dd            |S )NrA   r1   exc_infomessage)r   formatdict__dict__
getMessagepop)r   recordeir   ds        r   prepare_record_attributesz/_ForwardingLogHandler.prepare_record_attributes  su     _ 	$F##A !!$$&&%&	*	ir   c                p    |                      |          }| j                            | j        |           d S r   )r   r4   rC   r@   )r   r   
attributess      r   emitz_ForwardingLogHandler.emit  s5    33F;;
dj*55555r   )
rF   rG   rH   rI   r   NOTSETrv   r   r   r   r   s   @r   r   r     sl          -4N      
  &6 6 6 6 6 6 6r   r   c                  "    e Zd ZdZdddZd ZdS )	EnvironTNenvirondict | Nonec                T    |pi }d |                                 D             | _        d S )Nc                4    i | ]\  }}|t          |          S r   )r   ).0kvs      r   
<dictcomp>z$Environ.__init__.<locals>.<dictcomp>  s$    >>>da3q66>>>r   )itemsr   )r   r   s     r   rv   zEnviron.__init__  s,    -R>>gmmoo>>>r   c                H   K   |j                             | j                   d S r   )envupdater   rU   s     r   rN   zEnviron.setup  s$      	&&&&&r   r   )r   r   )rF   rG   rH   r.   rv   rN   r   r   r   r   r     sA        G? ? ? ? ?' ' ' ' 'r   r   c                  ,    e Zd ZdZdddd ffdZd ZdS )UploadDirectoryaU  A NannyPlugin to upload a local file to workers.

    Parameters
    ----------
    path: str
        A path to the directory to upload

    Examples
    --------
    >>> from distributed.diagnostics.plugin import UploadDirectory
    >>> client.register_worker_plugin(UploadDirectory("/path/to/directory"), nanny=True)  # doctest: +SKIP
    F)z.gitz.githubz.pytest_cachetestsdocsc                T    t           j                            |           d         dk    S )Nr   z.pyc)rm   rn   splitext)fns    r   <lambda>zUploadDirectory.<lambda>  s     ))"--a0F: r   c                j   t           j                            |          }t           j                            |          d         | _        || _        || _        dt           j                            |          d         z   | _        t          d          5 }t          j	        |dt          j
                  5 }t          j        |          D ]\  }}	|	D ]}
t           j                            ||
          t          fd|D                       r>                    t           j                  t          fd|D                       ryt           j                            t           j                            ||
          t           j                            |d                    }|                    |           	 d	d	d	           n# 1 swxY w Y   t#          |d
          5 }|                                | _        d	d	d	           n# 1 swxY w Y   d	d	d	           d	S # 1 swxY w Y   d	S )rj   zupload-directory-zip)	extensionwc              3  .   K   | ]} |          V  d S r   r   )r  	predicaterp   s     r   	<genexpr>z+UploadDirectory.__init__.<locals>.<genexpr>  s-      IIyyy22IIIIIIr   c              3      K   | ]}|v V  	d S r   r   )r  worddirss     r   r  z+UploadDirectory.__init__.<locals>.<genexpr>  s'      CCtt|CCCCCCr   z..Nrk   )rm   rn   
expandusersplitr.   update_pathr[   r	   zipfileZipFileZIP_DEFLATEDwalkjoinanyseprelpathwriterq   rr   rs   )r   rn   r.   r  
skip_wordsskipr  zrootfilesfilearchive_nameru   r  rp   s                @@r   rv   zUploadDirectory.__init__  s    w!!$''GMM$''+	&'"'--*=*=b*AA	u%%% 	%S'*>?? 81)+ 8 8%D$ % 8 8#%7<<d#;#;IIIIDIIIII %$'~~bf55CCCC
CCCCC %$')wGLLt44bgll46N6N( ( ,7777888 8 8 8 8 8 8 8 8 8 8 8 8 8 8 b$ %1FFHH	% % % % % % % % % % % % % % %!	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	%s[   !H(8D
GH(G	H(G	H(*HH(H	H(H	H((H,/H,c                v  K   t           j                            |j        dt	          j                     d          }t          |d          5 }|                    | j                   d d d            n# 1 swxY w Y   dd l	} |j
        |          5 }|                    |j                   d d d            n# 1 swxY w Y   | j        rXt           j                            |j        | j                  }|t          j        vr t          j                            d|           t          j        |           d S )Nztmp-z.zipwbr   )rn   )rm   rn   r$  local_directoryr_   r`   rq   r(  rs   r   r!  
extractallr  r   insertremove)r   rV   r  ru   r   r+  rn   s          r   rN   zUploadDirectory.setup  s     W\\%/1J
1J1J1JKK"d^^ 	qGGDI	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	W_R   	5ALLe3L444	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5  	)7<< 5tyAAD38##4(((
	"s$   A44A8;A8B;;B?B?N)rF   rG   rH   rI   rv   rN   r   r   r   r  r    sV           H::<#% #% #% #%J    r   r  c                  D    e Zd Zd Zd Zd Zd Zd Zd Zd Z	d Z
d	 Zd
S )forward_streamc                    || _         i | _        t          t          |          | _        |dk    rd| _        n!|dk    rd| _        nt          d| d          |dk    rdnd| _        g | _        d S )Nr   r   r      z1Expected stream to be 'stdout' or 'stderr'; got 'r   )_worker_original_methodsgetattrr   _stream_file
ValueError_buffer)r   streamr4   s      r   rv   zforward_stream.__init__  s    !#sF++XDJJxDJJMFMMM   !H,,QQ!
r   c                F    |                      |            ||           d S r   )_forward)r   write_fnrs   s      r   _writezforward_stream._write  s%    dr   c                v    | j                             |           d|v sd|v r|                                  d S d S )N
)r@  append_send)r   rs   s     r   rC  zforward_stream._forward!  sA    D!!!4<<44<<JJLLLLL (<r   c                l    | j         | j        ddd}| j                            d|           g | _         d S )N )r1   r.  r&  endprint)r@  r>  r:  rC   )r   rA   s     r   rJ  zforward_stream._send'  s:    |TZ2NNw,,,r   c                B    |                                    |             d S r   rJ  )r   flush_fns     r   _flushzforward_stream._flush,      






r   c                B    |                                    |             d S r   rP  )r   close_fns     r   _closezforward_stream._close0  rS  r   c                    t          | j        |          }|| j        |<   t          | j        |t	          j        ||                     d S r   )r<  r=  r;  setattr	functoolspartial)r   method_nameinterceptororiginal_methods       r   
_interceptzforward_stream._intercept4  sR    !$,<<.={+L+y'8o'V'V	
 	
 	
 	
 	
r   c                    |                      d| j                   |                      d| j                   |                      d| j                   | j        S )Nr(  flushr   )r^  rE  rR  rV  r=  r   s    r   	__enter__zforward_stream.__enter__;  sL    ---------|r   c                    | j                                          | j                                        D ]\  }}t	          | j         ||           i | _        d S r   )r=  r`  r;  r  rX  )r   exc_type	exc_value	tracebackattroriginals         r   __exit__zforward_stream.__exit__A  s]    "4::<< 	2 	2ND(DL$1111!#r   N)rF   rG   rH   rv   rE  rC  rJ  rR  rV  r^  ra  rh  r   r   r   r7  r7    s                 
    
 
 
  $ $ $ $ $r   r7  c                      e Zd ZdZd Zd ZdS )ForwardOutputa#  A Worker Plugin that forwards ``stdout`` and ``stderr`` from workers to clients

    This plugin forwards all output sent to ``stdout`` and ``stderr` on all workers
    to all clients where it is written to the respective streams. Analogous to the
    terminal, this plugin uses line buffering. To ensure that an output is written
    without a newline, make sure to flush the stream.

    .. warning::

        Using this plugin will forward **all** output in ``stdout`` and ``stderr`` from
        every worker to every client. If the output is very chatty, this will add
        significant strain on the scheduler. Proceed with caution!

    Examples
    --------
    >>> from dask.distributed import ForwardOutput
    >>> plugin = ForwardOutput()

    >>> client.register_worker_plugin(plugin)
    c                    t          j                    | _        | j                            t	          d|                     | j                            t	          d|                     d S )Nr   )r4   r   )
contextlib	ExitStack_exit_stackenter_contextr7  rM   s     r   rN   zForwardOutput.setup^  sa    %/11&&~hv'N'N'NOOO&&~hv'N'N'NOOOOOr   c                8    | j                                          d S r   )rn  r   rM   s     r   rP   zForwardOutput.teardownc  s         r   N)rF   rG   rH   rI   rN   rP   r   r   r   rj  rj  H  s?         *P P P
! ! ! ! !r   rj  )rX   rY   r   r   ),
__future__r   r   rl  rY  r   rm   r   r   r   r_   r   collections.abcr   typingr   r   r   
dask.utilsr   r	   distributed.schedulerr
   r   r   rF   r   r   rK   rS   ra   rc   ABCr}   r   r   r   r   Handlerr   r   r  r7  rj  r   r   r   <module>rx     s   " " " " " " 



          				      



   % % % % % % / / / / / / / / / / ( ( ( ( ( ( ( ( @????????		8	$	$f* f* f* f* f* f* f* f*RG G G G G G G GT N  N  N  N  N  N  N  NF	@ 	@ 	@ 	@N N N N N/ N N N o6 o6 o6 o6 o6\37 o6 o6 o6dI$ I$ I$ I$ I$> I$ I$ I$X>$ >$ >$ >$ >$ >$ >$ >$D4 4 4 4 4 4 4 4B#/ #/ #/ #/ #/< #/ #/ #/L(6 (6 (6 (6 (6GO (6 (6 (6V' ' ' ' 'k ' ' 'B B B B Bk B B BJ9$ 9$ 9$ 9$ 9$ 9$ 9$ 9$x! ! ! ! !L ! ! ! ! !r   