
    Lg                    x   d dl mZ d dlZd dlZd dlZd dlmZmZmZm	Z	m
Z
mZmZmZmZmZ d dlmZ d dlZd dlZd dlmZ d dlmZmZmZ d dlmZmZ d dlmZ d d	lm Z m!Z!m"Z" d d
l#m$Z$m%Z%m&Z& d dl'm(Z( d Z) G d de!      Z* G d de*      Z+ G d de!      Z,	 	 	 	 	 	 ddZ-ddZ. ej^                  e,      d        Z0y)    annotationsN)

CollectionHashable	ItemsViewIterableIteratorKeysViewMappingSequenceSet
ValuesView)Any)config)	clone_keyflattenis_dask_collection)keys_in_tasksreverse_dict)normalize_token)DaskCollectionGraphKey)ensure_dictimport_required	key_split)get_templatec                h     fd} j                         D ch c]  }|D ]  }|  }}} D ci c]  }|t                }} j                         D ]O  \  }}t        ||j	                         z
  |j                               D ]  }||   j                   ||              Q |S c c}}w c c}w )z'Returns the dependencies between layersc                r    j                         D ]  \  }}| |v s|c S  t        t        |        d      )Nz
 not found)itemsRuntimeErrorrepr)keykvlayerss      3lib/python3.12/site-packages/dask/highlevelgraph.py_find_layer_containing_keyz>compute_layer_dependencies.<locals>._find_layer_containing_key#   s;    LLNDAqax # d3i[
344    )valuessetr    r   keysadd)r&   r(   layerr#   all_keysr$   retr%   s   `       r'   compute_layer_dependenciesr1       s    5 "(BESEHB#
$V1ce8VC
$1 AFFH!4ahhjACFJJ1#67 B  J C
$s
   B)B/c                      e Zd ZU dZded<   ded<   	 	 d	 	 	 ddZej                  dd       Zej                  dd       Z		 	 	 	 	 	 dd	Z
dd
Z	 d	 	 	 	 	 	 	 ddZd ZddZd Zy)Layerax  High level graph layer

    This abstract class establish a protocol for high level graph layers.

    The main motivation of a layer is to represent a collection of tasks
    symbolically in order to speedup a series of operations significantly.
    Ideally, a layer should stay in this symbolic state until execution
    but in practice some operations will force the layer to generate all
    its internal tasks. We say that the layer has been materialized.

    Most of the default implementations in this class will materialize the
    layer. It is up to derived classes to implement non-materializing
    implementations.
    Mapping[str, Any] | Noner   collection_annotationsNc                    |xs& t        j                         j                         xs d| _        |xs) t        j                  t	        j
                  dd            | _        y)a  Initialize Layer object.

        Parameters
        ----------
        annotations : Mapping[str, Any], optional
            By default, None.
            Annotations are metadata or soft constraints associated with tasks
            that dask schedulers may choose to respect:
            They signal intent without enforcing hard constraints.
            As such, they are primarily designed for use with the distributed
            scheduler. See the dask.annotate function for more information.
        collection_annotations : Mapping[str, Any], optional. By default, None.
            Experimental, intended to assist with visualizing the performance
            characteristics of Dask computations.
            These annotations are *not* passed to the distributed scheduler.
        Nr5   )daskget_annotationscopyr   r   getr5   )selfr   r5   s      r'   __init__zLayer.__init__D   sP    * 'O$*>*>*@*E*E*GO4&< '
		JJ/6A
#r)   c                     y)z/Return whether the layer is materialized or notT r;   s    r'   is_materializedzLayer.is_materialized^   s     r)   c                "    | j                         S )a?  Return a set of all output keys

        Output keys are all keys in the layer that might be referenced by
        other layers.

        Classes overriding this implementation should not cause the layer
        to be materialized.

        Returns
        -------
        keys: Set
            All output keys
        r,   r?   s    r'   get_output_keyszLayer.get_output_keysc   s     yy{r)   c           	        t        |      t        |       k(  r1| | j                         D ci c]  }|| j                  ||       c}fS i }t               }i }|j	                         }|rf|j                         }| |   ||<   | j                  ||      ||<   ||   D ].  }||vs|| v s|j                  |       |j                  |       0 |rft        || j                        |fS c c}w )am  Remove unnecessary tasks from the layer

        In other words, return a new Layer with only the tasks required to
        calculate `keys` and a map of external key dependencies.

        Examples
        --------
        >>> inc = lambda x: x + 1
        >>> add = lambda x, y: x + y
        >>> d = MaterializedLayer({'x': 1, 'y': (inc, 'x'), 'out': (add, 'x', 10)})
        >>> _, deps = d.cull({'out'}, d.keys())
        >>> deps
        {'out': {'x'}, 'x': set()}

        Returns
        -------
        layer: Layer
            Culled layer
        deps: Map
            Map of external key dependencies
        r   )	lenr,   get_dependenciesr+   r9   popr-   MaterializedLayerr   )	r;   r,   all_hlg_keysr$   ret_depsseenoutworkds	            r'   cullz
Layer.cullt   s    2 t9D	! DHIIKPKqD))!\::KP 
 uyy{
A!WCF//<@HQKa[D=Dy	 !	  !$2B2BCXMM# Qs   C(c                "    t        || |   g      S )a(  Get dependencies of `key` in the layer

        Parameters
        ----------
        key:
            The key to find dependencies of
        all_hlg_keys:
            All keys in the high level graph.

        Returns
        -------
        deps: set
            A set of dependencies
        )r   )r;   r#   rI   s      r'   rF   zLayer.get_dependencies   s     \DI;77r)   c                   	
 ddl m} 	
fd	i }d}| j                         D ]9  \  }}|v r+t        |      }d
 	|      }|
r|j                  ||f}d}|||<   ; t        |      |fS )aC  Clone selected keys in the layer, as well as references to keys in other
        layers

        Parameters
        ----------
        keys
            Keys to be replaced. This never includes keys not listed by
            :meth:`get_output_keys`. It must also include any keys that are outside
            of this layer that may be referenced by it.
        seed
            Common hashable used to alter the keys; see :func:`dask.base.clone_key`
        bind_to
            Optional key to bind the leaf nodes to. A leaf node here is one that does
            not reference any replaced keys; in other words it's a node where the
            replacement graph traversal stops; it may still have dependencies on
            non-replaced nodes.
            A bound node will not be computed until after ``bind_to`` has been computed.

        Returns
        -------
        - New layer
        - True if the ``bind_to`` key was injected anywhere; False otherwise

        Notes
        -----
        This method should be overridden by subclasses to avoid materializing the layer.
        r   )chunksc                   t        |       }|t        u r.| r,t        | d         r| d   ft        fd| dd D              z   S |t        u r| D cg c]
  } |       c}S |t        u r*| j                         D ci c]  \  }}| |       c}}S 	 | vr| S 	 dt        |       S c c}w c c}}w # t        $ r | cY S w xY w)zhVariant of distributed.utils_comm.subs_multiple, which allows injecting
            bind_to
            r   c              3  .   K   | ]  } |        y wNr>   ).0iclone_values     r'   	<genexpr>z3Layer.clone.<locals>.clone_value.<locals>.<genexpr>   s     &Eu!{1~us      NF)typetuplecallablelistdictr    	TypeErrorr   )	otyprW   r$   r%   rX   is_leafr,   seeds	        r'   rX   z Layer.clone.<locals>.clone_value   s     q'Ce|hqtn!w&Equ&E!EEE0121A2267ggi@ida;q>)i@@}  %   D)) 3@
 ! Hs   B)<B.B4 4CCFT)dask.graph_manipulationrR   r    r   bindrH   )r;   r,   rd   bind_torR   dsk_newboundr#   valuerX   rc   s    ``      @@r'   clonezLayer.clone   s    B 	3	*, **,JCd{T*#E*&7#[[%9E E GCL ' !)500r)   c                    t        |       j                  | j                        }|j                  j	                  | j                         |S )z#Default shallow copy implementation)r[   __new__	__class____dict__update)r;   objs     r'   __copy__zLayer.__copy__   s5    4j  0DMM*
r)   c           	        |dk7  rt        |      }n8t        | d      rt        | j                        }n| j                  j                  }d}| j
                  rI| j
                  j                  d      dk(  r+| j
                  j                  d      }|rddlm}  ||      }t        d      j                  | j                         |||| j                         ||	      S )
N namer[   dask.array.core.ArrayrR   r   )svgzhighlevelgraph_layer.html.j2)materialized	shortnamelayer_indexhighlevelgraph_keyinfodependenciessvg_repr)r   hasattrru   rn   __name__r5   r:   dask.array.svgrw   r   renderr@   layer_info_dict)r;   rz   r{   r}   ry   r~   rR   rw   s           r'   _repr_html_zLayer._repr_html_  s    #!"45IT6"!$)),I//I''++//7;RR0044X>F.v;:;BB--/#1%%'% C 
 	
r)   c                   t        |       j                  | j                         t        | j	                                d}| j
                  C| j
                  j                         D ]&  \  }}t        j                  t        |            ||<   ( | j                  I| j                  j                         D ],  \  }}|dk7  st        j                  t        |            ||<   . |S )N)
layer_typer@   znumber of outputsrR   )r[   r   r@   rE   rC   r   r    htmlescapestrr5   )r;   r|   r#   vals       r'   r   zLayer.layer_info_dict#  s    t*--#335$'(<(<(>$?#@

 ' ,,224S KKC1S	 5&&2 77==?S(? $CH 5DI @ r)   NN)r   r4   r5   r4   )returnbool)r   zSet[Key])r,   set[Key]rI   Collection[Key]r   z$tuple[Layer, Mapping[Key, set[Key]]])r#   r   rI   r   r   r+   rU   )r,   r+   rd   r   rg   z
Key | Noner   ztuple[Layer, bool])rt   rt   r>   )r   
__module____qualname____doc____annotations__r<   abcabstractmethodr@   rC   rO   rF   rk   rr   r   r   r>   r)   r'   r3   r3   1   s     *)44 15;?
-
 !9
4 	  	  .N.N,;.N	-.N`8* #	I1I1 I1 	I1
 
I1V
:r)   r3   c                  J     e Zd ZdZd	d
 fdZd Zd Zd Zd Zd Z	d Z
 xZS )rH   zFully materialized layer of `Layer`

    Parameters
    ----------
    mapping: Mapping
        The mapping between keys and tasks, typically a dask graph.
    c                6    t         |   ||       || _        y )N)r   r5   )superr<   mapping)r;   r   r   r5   rn   s       r'   r<   zMaterializedLayer.__init__=  s$    #<R 	 	
 r)   c                    || j                   v S rU   r   r;   r$   s     r'   __contains__zMaterializedLayer.__contains__C  s    DLL  r)   c                     | j                   |   S rU   r   r   s     r'   __getitem__zMaterializedLayer.__getitem__F  s    ||Ar)   c                ,    t        | j                        S rU   )iterr   r?   s    r'   __iter__zMaterializedLayer.__iter__I  s    DLL!!r)   c                ,    t        | j                        S rU   )rE   r   r?   s    r'   __len__zMaterializedLayer.__len__L  s    4<<  r)   c                     y)NTr>   r?   s    r'   r@   z!MaterializedLayer.is_materializedO  s    r)   c                "    | j                         S rU   rB   r?   s    r'   rC   z!MaterializedLayer.get_output_keysR  s    yy{r)   r   )r   r   )r   r   r   r   r<   r   r   r   r   r@   rC   __classcell__)rn   s   @r'   rH   rH   4  s+    !"!r)   rH   c                  :   e Zd ZU dZded<   ded<   ded<   ded	<   d
ed<   	 d#	 	 	 	 	 d$dZed        Ze	 d%	 	 	 	 	 	 	 d&d       Zd'dZ	d(dZ
d)dZd*dZd+dZd,dZd-dZd.dZd/dZed0d       Zd1dZed2d       Zd3dZd4dZd5dZd6dZd7d Zd8d!Zd8d"Zy)9HighLevelGrapha
  Task graph composed of layers of dependent subgraphs

    This object encodes a Dask task graph that is composed of layers of
    dependent subgraphs, such as commonly occurs when building task graphs
    using high level collections like Dask array, bag, or dataframe.

    Typically each high level array, bag, or dataframe operation takes the task
    graphs of the input collections, merges them, and then adds one or more new
    layers of tasks for the new operation.  These layers typically have at
    least as many tasks as there are partitions or chunks in the collection.
    The HighLevelGraph object stores the subgraphs for each operation
    separately in sub-graphs, and also stores the dependency structure between
    them.

    Parameters
    ----------
    layers : Mapping[str, Mapping]
        The subgraph layers, keyed by a unique name
    dependencies : Mapping[str, set[str]]
        The set of layers on which each layer depends
    key_dependencies : dict[Key, set], optional
        Mapping (some) keys in the high level graph to their dependencies. If
        a key is missing, its dependencies will be calculated on-the-fly.

    Examples
    --------
    Here is an idealized example that shows the internal state of a
    HighLevelGraph

    >>> import dask.dataframe as dd

    >>> df = dd.read_csv('myfile.*.csv')  # doctest: +SKIP
    >>> df = df + 100  # doctest: +SKIP
    >>> df = df[df.name == 'Alice']  # doctest: +SKIP

    >>> graph = df.__dask_graph__()  # doctest: +SKIP
    >>> graph.layers  # doctest: +SKIP
    {
     'read-csv': {('read-csv', 0): (pandas.read_csv, 'myfile.0.csv'),
                  ('read-csv', 1): (pandas.read_csv, 'myfile.1.csv'),
                  ('read-csv', 2): (pandas.read_csv, 'myfile.2.csv'),
                  ('read-csv', 3): (pandas.read_csv, 'myfile.3.csv')},
     'add': {('add', 0): (operator.add, ('read-csv', 0), 100),
             ('add', 1): (operator.add, ('read-csv', 1), 100),
             ('add', 2): (operator.add, ('read-csv', 2), 100),
             ('add', 3): (operator.add, ('read-csv', 3), 100)}
     'filter': {('filter', 0): (lambda part: part[part.name == 'Alice'], ('add', 0)),
                ('filter', 1): (lambda part: part[part.name == 'Alice'], ('add', 1)),
                ('filter', 2): (lambda part: part[part.name == 'Alice'], ('add', 2)),
                ('filter', 3): (lambda part: part[part.name == 'Alice'], ('add', 3))}
    }

    >>> graph.dependencies  # doctest: +SKIP
    {
     'read-csv': set(),
     'add': {'read-csv'},
     'filter': {'add'}
    }

    See Also
    --------
    HighLevelGraph.from_collections :
        typically used by developers to make new HighLevelGraphs
    zMapping[str, Layer]r&   Mapping[str, set[str]]r}   dict[Key, set[Key]]key_dependenciesr_   _to_dictr+   _all_external_keysNc           	         || _         |xs i | _        |j                         D ci c]#  \  }}|t        |t              r|n
t        |      % c}}| _        y c c}}w rU   )r}   r   r    
isinstancer3   rH   r&   )r;   r&   r}   r   r$   r%   s         r'   r<   zHighLevelGraph.__init__  s_     ) 0 6B 
&1 Jq%(q.?.BB&
 
s   (Ac                r   t        |      st        t        |            |j                         }t	        |t
              rPt        |j                  d      }|||<   t        |j                  d      }t        |j                               ||<   n t        |      }||||i}||h|t               i} | ||      S )z4`from_collections` optimized for a single collectionTr9   )r   r`   r[   __dask_graph__r   r   r   r&   r}   r+   __dask_layers___get_some_layer_name)clsru   r.   
collectiongraphr&   depsr#   s           r'   _from_collectionzHighLevelGraph._from_collection  s     "*-D,--))+e^, D9F F4Lu11=DZ779:DJ&z2CE3.F3%ce,D64  r)   c                @   t        |      dk(  r| j                  |||d         S ||i}t               }||i}t        j                  |t
              D ]  }t        |      r|j                         }t        |t              rS|j                  |j                         |j                  |j                         |t        |j                               z  }t        |      }	|||	<   |j                  |	       t               ||	<   t!        t#        |              | ||      S )ac  Construct a HighLevelGraph from a new layer and a set of collections

        This constructs a HighLevelGraph in the common case where we have a single
        new layer and a set of old collections on which we want to depend.

        This pulls out the ``__dask_layers__()`` method of the collections if
        they exist, and adds them to the dependencies for this new layer.  It
        also merges all of the layers from all of the dependent collections
        together into the new layers for this graph.

        Parameters
        ----------
        name : str
            The name of the new layer
        layer : Mapping
            The graph layer itself
        dependencies : List of Dask collections
            A list of other dask collections (like arrays or dataframes) that
            have graphs themselves

        Examples
        --------

        In typical usage we make a new task layer, and then pass that layer
        along with all dependent collections to this method.

        >>> def add(self, other):
        ...     name = 'add-' + tokenize(self, other)
        ...     layer = {(name, i): (add, input_key, other)
        ...              for i, input_key in enumerate(self.__dask_keys__())}
        ...     graph = HighLevelGraph.from_collections(name, layer, dependencies=[self])
        ...     return new_collection(name, graph)
        rZ   r   )r#   )rE   r   r+   toolzuniqueidr   r   r   r   rp   r&   r}   r   r   r-   r`   r[   )
r   ru   r.   r}   r&   name_depr   r   r   r#   s
             r'   from_collectionszHighLevelGraph.from_collections  s    P |!''e\!_EE U%)8$4,,|<J!*-"113e^4MM%,,/KK 2 23J$>$>$@ AAH.z:C"'F3KLL% #DIZ 011 = 64  r)   c                &   	 | j                   |   |   S # t        $ r Y nw xY w	 | j                   |d      |   S # t        t        t        f$ r Y nw xY w| j                   j	                         D ]  }	 ||   c S # t        $ r Y w xY w t        |      )Nr   )r&   KeyError
IndexErrorr`   r*   )r;   r#   rN   s      r'   r   zHighLevelGraph.__getitem__  s    	;;s#C(( 			;;s1v&s++*i0 		 ##%Av  & sms-    	  9 AA1A88	BBc                V    t        d | j                  j                         D              S )Nc              3  2   K   | ]  }t        |        y wrU   )rE   )rV   r.   s     r'   rY   z)HighLevelGraph.__len__.<locals>.<genexpr>  s     @+?%3u:+?s   )sumr&   r*   r?   s    r'   r   zHighLevelGraph.__len__  s"     @4;;+=+=+?@@@r)   c                4    t        | j                               S rU   )r   to_dictr?   s    r'   r   zHighLevelGraph.__iter__  s    DLLN##r)   c                b    	 | j                   S # t        $ r t        |       x}| _         |cY S w xY w)zIEfficiently convert to plain dict. This method is faster than dict(self).)r   AttributeErrorr   )r;   rL   s     r'   r   zHighLevelGraph.to_dict  s6    	==  	"-d"33C$-J	s    ..c                >    | j                         j                         S )zGet all keys of all the layers.

        This will in many cases materialize layers, which makes it a relatively
        expensive operation. See :meth:`get_all_external_keys` for a faster alternative.
        )r   r,   r?   s    r'   r,   zHighLevelGraph.keys&  s     ||~""$$r)   c                    	 | j                   S # t        $ rT t               }| j                  j	                         D ]!  }|j                  |j                                # || _         |cY S w xY w)zGet all output keys of all layers

        This will in most cases _not_ materialize any layers, which makes
        it a relative cheap operation.

        Returns
        -------
        keys: set
            A set of all external keys
        )r   r   r+   r&   r*   rp   rC   )r;   r,   r.   s      r'   get_all_external_keysz$HighLevelGraph.get_all_external_keys.  sf    
	*** 	D++- E1134	 .
 '+D#K	s    AA+*A+c                >    | j                         j                         S rU   )r   r    r?   s    r'   r    zHighLevelGraph.itemsE  s    ||~##%%r)   c                >    | j                         j                         S rU   )r   r*   r?   s    r'   r*   zHighLevelGraph.valuesH  s    ||~$$&&r)   c                $   | j                         }|| j                  j                         z
  }|rV| j                  j                         D ]9  }||j                         z  D ]!  }|j	                  ||      | j                  |<   # ; | j                  S )zGet dependencies of all keys

        This will in most cases materialize all layers, which makes
        it an expensive operation.

        Returns
        -------
        map: Mapping
            A map that maps each key to its dependencies
        )r,   r   r&   r*   rF   )r;   r/   missing_keysr.   r$   s        r'   get_all_dependenciesz#HighLevelGraph.get_all_dependenciesK  s     99;$"7"7"<"<">>++-%

4A/4/E/Ea/RD))!, 5 . $$$r)   c                ,    t        | j                        S rU   )r   r}   r?   s    r'   
dependentszHighLevelGraph.dependents^  s    D--..r)   c                    t        t        | j                  d      t        | j                  d      | j                  j                               S )NTr   )r   r   r&   r}   r   r9   r?   s    r'   r9   zHighLevelGraph.copyb  s?    $/))5!!&&(
 	
r)   c                Z   i }i }|D ]  }t        |t              r7|j                  |j                         |j                  |j                         Jt        |t
              r7||t        t        |            <   t               |t        t        |            <   t        |        | ||      S rU   )
r   r   rp   r&   r}   r   r   r   r+   r`   )r   graphsr&   r}   gs        r'   mergezHighLevelGraph.mergei  s    #%,.A!^,ahh'##ANN3Aw'%&s2a5z"+.5SAZ(l"  6<((r)   c                >    ddl m} t        | fi |} ||||       |S )a?  
        Visualize this dask high level graph.

        Requires ``graphviz`` to be installed.

        Parameters
        ----------
        filename : str or None, optional
            The name of the file to write to disk. If the provided `filename`
            doesn't include an extension, '.png' will be used by default.
            If `filename` is None, no file will be written, and the graph is
            rendered in the Jupyter notebook only.
        format : {'png', 'pdf', 'dot', 'svg', 'jpeg', 'jpg'}, optional
            Format in which to write output file. Default is 'svg'.
        color : {None, 'layer_type'}, optional (default: None)
            Options to color nodes.
            - None, no colors.
            - layer_type, color nodes based on the layer type.
        **kwargs
           Additional keyword arguments to forward to ``to_graphviz``.

        Examples
        --------
        >>> x.dask.visualize(filename='dask.svg')  # doctest: +SKIP
        >>> x.dask.visualize(filename='dask.svg', color='layer_type')  # doctest: +SKIP

        Returns
        -------
        result : IPython.display.Image, IPython.display.SVG, or None
            See dask.dot.dot_graph for more information.

        See Also
        --------
        dask.dot.dot_graph
        dask.base.visualize # low level variant
        r   )graphviz_to_file)dask.dotr   to_graphviz)r;   filenameformatkwargsr   r   s         r'   	visualizezHighLevelGraph.visualizex  s(    L 	.''Hf-r)   c                D   | j                   j                         D ci c]  \  }}|t        |       }}}| j                   D ci c]  }|g  }}g }| j                   j                         D ]4  \  }}|D ]  }||   j                  |        |r$|j                  |       6 g }t        |      dkD  ra|j	                         }|j                  |       ||   D ])  }	||	xx   dz  cc<   ||	   dk(  s|j                  |	       + t        |      dkD  ra|S c c}}w c c}w )a  Sort the layers in a high level graph topologically

        Parameters
        ----------
        hlg : HighLevelGraph
            The high level graph's layers to sort

        Returns
        -------
        sorted: list
            List of layer names sorted topologically
        r   rZ   )r}   r    rE   appendrG   )
r;   r$   r%   degreereverse_depsreadydepr0   r.   rdeps
             r'   _toposort_layerszHighLevelGraph._toposort_layers  s    )-(9(9(?(?(AB(A1!SV)(AB=A=N=N-O=Nae=N-O%%++-DAqS!((+ Q	 .
 %j1nIIKEJJu$U+t!$<1$LL& , %j1n 
! C-Os   D
Dc                   ddl m} t        t        |            }| j	                         }i }i }t        | j                               D ]  }| j                  |   }|j                  |j                               }	|	s4|j                  |	|      \  }
}t               }|j                         D ]  }||z  }	 ||
j                         z  }||z  }|
||<   t        ||      s9t        |t              s)|j                         st        |      t        |      k(  s|j!                  |        t        |j#                               }|D ci c]  }|| j$                  |   |z   }}t'        |||      S c c}w )a  Return new HighLevelGraph with only the tasks required to calculate keys.

        In other words, remove unnecessary tasks from dask.

        Parameters
        ----------
        keys
            iterable of keys or nested list of keys such as the output of
            ``__dask_keys__()``

        Returns
        -------
        hlg: HighLevelGraph
            Culled high level graph
        r   )	Blockwise)dask.layersr   r+   r   r   reversedr   r&   intersectionrC   rO   r*   r   rH   r@   rE   rp   r,   r}   r   )r;   r,   r   keys_setall_ext_keys
ret_layersret_key_deps
layer_namer.   output_keysculled_layerculled_depsexternal_depsrN   ret_layers_keysret_dependenciess                   r'   rO   zHighLevelGraph.cull  sy     	*wt}%113
"4#8#8#:;JKK
+E #//0E0E0GHK,1JJ{L,Q)k !$$++-A!Q&M .!=!=!??M) *6
:&ui0!%):;--/SZ3{CS5S !''4E <L joo/0 )
(
 ))*5GG( 	 

 j*:LII
s   E0c                    t        |      }i }i }|rP|j                         }| j                  |   ||<   | j                  |   ||<   |||   |j	                         z
  z  }|rPt        ||      S )a  Return a new HighLevelGraph with only the given layers and their
        dependencies. Internally, layers are not modified.

        This is a variant of :meth:`HighLevelGraph.cull` which is much faster and does
        not risk creating a collision between two layers with the same name and
        different content when two culled graphs are merged later on.

        Returns
        -------
        hlg: HighLevelGraph
            Culled high level graph
        )r+   rG   r&   r}   r,   r   )r;   r&   to_visitr   r   r$   s         r'   cull_layerszHighLevelGraph.cull_layers  s     v;
A KKNJqM"&"3"3A"6Q(+.>.C.C.EEEH	  j*:;;r)   c                   | j                   j                         D ]W  \  }}|| j                  vrt        dt	        |       d      |D ]'  }|| j                   vst        t	        |       d       Y | j                  j                         D ]  }t        |d      rJ  t        | j                        }| j                   j                         }|j                         }||k7  r#t        dt        |      dt        |            |D ]W  }| j                   |   ||   k7  st        dt	        |       dt	        | j                   |          dt	        ||          d	       y )
Nzdependencies[z] not found in layersz not found in dependenciesr   zincorrect dependencies keys z
 expected zincorrect HLG dependencies[z]: z from task dependencies)
r}   r    r&   
ValueErrorr"   r*   r   r1   r,   r+   )	r;   r   r   r   r.   r}   dep_key1dep_key2r$   s	            r'   validatezHighLevelGraph.validate   s    $ 1 1 7 7 9J, #D$4#55JK  d///$S	{2L%MNN  !: [['')E5-000 * 2$++> $$))+$$&x.s8}.? @M,.  A  #|A6 1$q'#d4CTCTUVCW>X=Y Z  $\!_ 566MO  r)   c                N   t        |       j                   dt        | j                         d}|d| j                  j
                   d| j                  j                   dt        t        |              dz  }t        | j                               D ]  \  }}|d| d| d	z  } |S )
Nz with z	 layers.
<.z object at z>
 z. 
)
r[   r   rE   r&   rn   r   hexr   	enumerater   )r;   representationrW   layerkeys       r'   __repr__zHighLevelGraph.__repr__B  s     J//0s4;;7G6H
SAdnn778$..:Q:Q9RR]^abdeibj^k]llopp$T%:%:%<=KAx!Bxj33N >r)   c           
         t        d      j                  t        |       j                  | j                  | j                         | j                  t        | j                                     S )Nzhighlevelgraph.html.j2)r[   r&   toposortlayer_dependencies	n_outputs)	r   r   r[   r   r&   r   r}   rE   r   r?   s    r'   r   zHighLevelGraph._repr_html_I  sZ    45<<d$$;;**,#00$4467 = 
 	
r)   rU   )r&   zMapping[str, Graph]r}   r   r   zdict[Key, set[Key]] | None)r>   )ru   r   r.   r   r}   zSequence[DaskCollection]r   r   )r#   r   r   r   )r   int)r   zIterator[Key])r   zdict[Key, Any])r   r
   )r   r   )r   zItemsView[Key, Any])r   zValuesView[Any])r   r   )r   zdict[str, set[str]])r   r   )r   r   r   r   )zdask-hlg.svgN)r   z	list[str])r,   zIterable[Key]r   r   )r&   zIterable[str]r   r   )r   Noner   r   )r   r   r   r   r   r<   classmethodr   r   r   r   r   r   r,   r   r    r*   r   propertyr   r9   r   r   r   rO   r   r  r  r   r>   r)   r'   r   r   V  s$   ?B  (())N 8<	
#
 -
 5	
 ! !$ 
 24	;!;! ;! /	;!
 
;! ;!z,A$%.&'%& / /
 ) )*X>CJJ<0 D
r)   r   c                   ddl m}m}	 t        dd      }
|xs i }|xs i }|xs i }|xs i }|xs i }||d<   d|d<   d|d	<   |j	                  |       |
j                  |||
      }i }| j                  D ]  }t        | j                  |         ||<    t        |j                               }t        |j                               }i }|j                  d      }|dk(  rddgddgddgddgddgddgddgddgd}| j                  D ]`  } |	|      }|j                  |i       } |||      }||k(  rdnt        d||   |z
  ||z
  z  dz  z         }t        t        | j                  |         j                         }d|j#                  dd       d||    d}| j                  |   j$                  }|r|j                  d      dk(  rN|d|j                  d       d|j                  d        d!|j                  d"       d#|j                  d$       d%	z  }|j                  d      d&k(  rd'd(i}|j                  d)      }|d*|j                  d+       d,|j                  |j                  d-             d%t        |       d.t        t        |            d/k  rt        |      nd0 d%	z  }|j'                  d1t        |             |j'                  d2t        |             |j'                  d3t        |             |dk(  rUj                  |      d   }d4|j                  |      d5<   |j'                  d6t        |             |j'                  d7d8        |j(                  |fi | c | j                  j+                         D ].  \  }} |	|      }|D ]  } |	|      }|j-                  ||        0 |dk(  rd9} d:}!j+                         D ]  \  }}|d5   s|!d;|d    d<| d=z  }! |!d>z  }!|j                  | i       }|j'                  d1t        |!             |j'                  d2d?       |j'                  d@dA        |j(                  | fi | |S )BNr   )labelru   graphvizaJ  Drawing dask graphs with the graphviz visualization engine requires the `graphviz` python library and the `graphviz` system library.

Please either conda or pip install as follows:

  conda install python-graphviz     # either conda install
  python -m pip install graphviz    # or pip install and follow installation instructionsrankdirboxshape	helveticafontname)
graph_attr	node_attr	edge_attrcolorr   z#CCC7F9Fz#F9CCC7z#FFD9F2z#D9F2FFz#D9FFE6z#DBDEE5)DataFrameIOLayerShuffleLayerSimpleShuffleLayerArrayOverlayLayerBroadcastJoinLayerr   BlockwiseLayerrH   )cache   zA r3   rt   z Layer with z Tasks.
r[   rv   zArray Shape: z
Data Type: dtypez
Chunk Size: 	chunksizez
Chunk Type: 
chunk_typer  zdask.dataframe.core.DataFramezpandas.core.frame.DataFramepandascolumnszNumber of Partitions: npartitionsz
DataFrame Type: dataframe_typez DataFrame Columns: (   z[...]r  fontsizetooltipTrZ   	fillcolorstylefilledr   zn<<TABLE BORDER="0" CELLBORDER="1" CELLSPACING="0" CELLPADDING="5"><TR><TD><B>Legend: Layer types</B></TD></TR>z<TR><TD BGCOLOR="z">z
</TD></TR>z	</TABLE>>20margin0)r   r  ru   r   rp   Digraphr}   rE   r&   minr*   maxr:   r  r   r[   r   replacer5   
setdefaultnoder    edge)"hgdata_attributesfunction_attributesr  r  r  r  r   r  ru   r  r   n_tasksr.   	min_tasks	max_tasksr'  r   layer_colorsr   attrs
node_label	node_sizer   node_tooltipslayer_cadftypecols
node_colorr   r   dep_namelegend_titlelegend_labels"                                     r'   r   r   S  s    %	dH &+O-3!rJRIRI#JyIg'Ijfi 	 	A GRYYu-. ! GNN$%IGNN$%IEJJwE!*E 2&.#,e"4"+U!3#,e"4#U+(%0"+U!3	
 %[
##E2.5.
 I% RGENY69y;PQUWWWX 	 bii./889
##GR01gen=MYW 	 99U#::||F#'>>#HLL$9#: ;""*,,w"7!8 9##+<<#<"= >##+<<#=">bB ||F#'FF7B||I.,X\\--H,I J''-zz(,,?O2P'Q&RRT4yk!53s4y>UWCWc$i]d5eegi 	#j/2S^4C$67L %))*5a8J.2LZ(+[#j/:Wh/z#U#_ !b ,,.t%[
CCyHFF8Z(  / ; 	
 ".!3!3!5JQx"3E!H:R
|: VV "6 	###L"5#l"34T*3'|%u%Hr)   c                ~    	 | j                         \  }|S # t        t        f$ r t        t	        |             cY S w xY w)zLSomehow get a unique name for a Layer from a non-HighLevelGraph dask mapping)r   r   r   r   r   )r   ru   s     r'   r   r     sA    #,,.J' # 2j>""#s    #<<c                Z    t        t        | j                  j                                     S rU   )r   r^   r&   r,   )hlgs    r'   register_highlevelgraphrU    s    4

 1233r)   )NNBTNNNr  )1
__future__r   r   r9   r   collections.abcr   r   r   r   r	   r
   r   r   r   r   typingr   tlzr   r7   r   	dask.baser   r   r   	dask.corer   r   dask.tokenizer   dask.typingr   r   r   
dask.utilsr   r   r   dask.widgetsr   r1   r3   rH   r   r   r   registerrU  r>   r)   r'   <module>rb     s    " 
         < < 1 ) 2 2 > > %"@E @F Dz
U z
~ FR# .)4 *4r)   