
    Lg~                    8   d dl mZ d dlZd dlZd dlmZmZmZmZ d dl	m
Z
mZmZ d dlmZ d dlmZ d dlmZ d dlZd dlZd dlmZmZ d d	lmZmZmZmZmZmZ d d
lmZm Z m!Z! d dl"m#Z#m$Z$m%Z%m&Z& d dl'm(Z(m)Z) d dl*m+Z+m,Z, d dl-m.Z. d dl/m0Z0 d dl1m2Z2m3Z3m4Z4  G d d      Z5 G d de5      Z6 G d de5      Z7d Z8d Z9dZ:e:fdZ;ddddddZ< G d de,      Z=d  Z>dddddddd!d"Z?d0d#Z@d$ ZAd1d%ZBd1d&ZCd' ZDd2d(ZEd3d)ZFd* ZG e2       d+        ZHd4d,ZId- ZJd5d.ZKd/ ZLy)6    annotationsN)HashableIterableMappingSequence)fieldsis_dataclassreplace)product)prod)Any)baseutils)Dict	GraphNodeListTaskTaskRefparse_input)	clone_keyget_name_from_keytokenize)flatten
ishashablekeys_in_tasksreverse_dict)Delayedfinalize)HighLevelGraphLayer)fuse)Key)_deprecatedensure_dicthomogeneous_deepmapc                  P    e Zd ZU dZded<   ded<   ddZddZedd       Zdd	Z	y
)BlockwiseDepa  Blockwise-IO argument

    This is the base class for indexable Blockwise-IO arguments.
    When constructing a ``Blockwise`` Layer, one or more of the
    collection tuples passed in with ``indices`` may contain a
    ``BlockwiseDep`` instance (in place of a "real" collection name).
    This allows a new collection to be created (via IO) within a
    ``Blockwise`` layer.

    Parameters
    ----------
    numblocks: tuple[int, ...]
        The number of blocks/partitions the object can support
        along each dimension.
    produces_tasks: bool
        Whether any nested tasks will be passed to the Blockwise
        function.

    See Also
    --------
    dask.blockwise.Blockwise
    dask.blockwise.BlockwiseDepDict
    tuple[int, ...]	numblocksboolproduces_tasksc                    t        d      )z8Return Blockwise-function arguments for a specific indexz6Must define `__getitem__` for `BlockwiseDep` subclass.)NotImplementedErrorselfidxs     .lib/python3.12/site-packages/dask/blockwise.py__getitem__zBlockwiseDep.__getitem__5   s    !D
 	
    c                H    	 | j                  |      S # t        $ r |cY S w xY w)z$BlockwiseDep ``__getitem__`` Wrapper)r3   KeyError)r0   r1   defaults      r2   getzBlockwiseDep.get;   s,    	##C(( 	N	s    !!c                     y)a0  Whether this object will produce external key dependencies.

        An external key corresponds to a task key or ``Delayed``-object
        key that does not originate from within the ``Blockwise`` layer
        that is including this ``BlockwiseDep`` object in its ``indices``.
        A ``BlockwiseDep`` object should only return external-key
        dependencies when those dependencies do not correspond to a
        blockwise-compatible Dask collection (otherwise the collection
        name should just be included in ``indices`` list instead).
        F r0   s    r2   produces_keyszBlockwiseDep.produces_keysB   s     r4   c                N    dt        |       j                   d| j                   dS )N< >)type__name__r*   r;   s    r2   __repr__zBlockwiseDep.__repr__P   s'    4:&&'q(8::r4   Nr1   r)   returnr   rE   r+   )rE   str)
rB   
__module____qualname____doc____annotations__r3   r8   propertyr<   rC   r:   r4   r2   r(   r(      s8    0 
  ;r4   r(   c                  P    e Zd ZdZ	 	 	 d	 	 	 	 	 	 	 ddZed	d       Zd
dZddZy)BlockwiseDepDictaH  Dictionary-based Blockwise-IO argument

    This is a dictionary-backed instance of ``BlockwiseDep``.
    The purpose of this class is to simplify the construction
    of IO-based Blockwise Layers with block/partition-dependent
    function arguments that are difficult to calculate at
    graph-materialization time.

    Examples
    --------

    Specify an IO-based function for the Blockwise Layer. Note
    that the function will be passed a single input object when
    the task is executed (e.g. a single ``tuple`` or ``dict``):

    >>> import pandas as pd
    >>> func = lambda x: pd.read_csv(**x)

    Use ``BlockwiseDepDict`` to define the input argument to
    ``func`` for each block/partition:

    >>> dep = BlockwiseDepDict(
    ...     mapping={
    ...         (0,) : {
    ...             "filepath_or_buffer": "data.csv",
    ...             "skiprows": 1,
    ...             "nrows": 2,
    ...             "names": ["a", "b"],
    ...         },
    ...         (1,) : {
    ...             "filepath_or_buffer": "data.csv",
    ...             "skiprows": 3,
    ...             "nrows": 2,
    ...             "names": ["a", "b"],
    ...         },
    ...     }
    ... )

    Construct a Blockwise Layer with ``dep`` specified
    in the ``indices`` list:

    >>> layer = Blockwise(
    ...     output="collection-name",
    ...     output_indices="i",
    ...     task=Task("collection-name", func, TaskRef("_0")),
    ...     indices=[(dep, "i")],
    ...     numblocks={},
    ... )

    See Also
    --------
    dask.blockwise.Blockwise
    dask.blockwise.BlockwiseDep
    Nc                X    || _         || _        |xs t        |      f| _        || _        y N)mappingr,   lenr*   _produces_keys)r0   rQ   r*   r,   r<   s        r2   __init__zBlockwiseDepDict.__init__   s1     , #5s7|o ,r4   c                    | j                   S rP   )rS   r;   s    r2   r<   zBlockwiseDepDict.produces_keys   s    """r4   c                    	 | j                   |   S # t        $ rA}|d t        | j                         }|| j                   v r| j                   |   cY d }~S |d }~ww xY wrP   )rQ   r6   rR   r*   )r0   r1   errflat_idxs       r2   r3   zBlockwiseDepDict.__getitem__   s^    	<<$$ 
	 0S01H4<<'||H--I
	s     	A4AAAAc                ,    t        | j                        S rP   )rR   rQ   r;   s    r2   __len__zBlockwiseDepDict.__len__   s    4<<  r4   )NFF)rQ   dictr*   ztuple[int, ...] | Noner,   r+   r<   r+   rF   rD   rE   int)	rB   rH   rI   rJ   rT   rL   r<   r3   rZ   r:   r4   r2   rN   rN   T   s\    5t -1$#,, *, 	,
 ," # #!r4   rN   c                  0    e Zd ZU dZdZded<   ddZd	dZy)

BlockIndexzIndex BlockwiseDep argument

    The purpose of this class is to provide each
    block of a ``Blockwise``-based operation with
    the current block index.
    Fr+   r,   c                    || _         y rP   )r*   )r0   r*   s     r2   rT   zBlockIndex.__init__   s     #r4   c                    |S rP   r:   r/   s     r2   r3   zBlockIndex.__getitem__   s    
r4   N)r*   r)   )r1   r)   rE   r)   )rB   rH   rI   rJ   r,   rK   rT   r3   r:   r4   r2   r_   r_      s     !ND #
r4   r_   c           	     X   t        | t              r.| j                         D ci c]  \  }}|t        ||       c}}S t	        |       t
        t        t        fv r) t	        |       | D cg c]  }t        ||       c}      S 	 ||    S c c}}w c c}w # t        t        f$ r | cY S w xY w)zCreate a new task with the values substituted

    This is like dask.core.subs, but takes a dict of many substitutions to
    perform simultaneously.  It is not as concerned with micro performance.
    )

isinstancer[   itemssubsrA   tuplelistsetr6   	TypeError)tasksubstitutionkvxs        r2   re   re      s     $59ZZ\B\TQ4<((\BBDzeT3''tDz$?$Q4<0$?@@D!!	 C? i  s   B
*BB B)(B)c                4    | | S t        fd| D              S )z<A simple subs function that works both on tuples and stringsc              3  B   K   | ]  }j                  ||        y wrP   )r8   ).0crk   s     r2   	<genexpr>zindex_subs.<locals>.<genexpr>   s     9S\%%a+Ss   )rf   )indrk   s    `r2   
index_subsru      s    
{
9S999r4   __dask_blockwise__c                    |d| z  z   S )Nz%dr:   )iprefixs     r2   blockwise_tokenrz      s    D1Hr4   r:   )r*   concatenatenew_axesdependenciesc                  |xs i }t        |      }|ddd   D 	
ch c]  }	|	|	D ]  }
|
  c}
}	t        |      z  }t        t        |            D 
ci c]  \  }
}|t	        |
d       }}
}t        t        |      |      }g }|ddd   D ]-  }|t        |      }n|}|j                  t        ||             / ||ddd<   |j                         D ci c]  \  }}t        |f|      d   | }}}g }g }t        j                  d|      D ]'  \  }}|j                  |       |j                  |       ) t        t        t        t        t        t        t        |                              }|D ch c]  }|j                         D ]  }|  }}}|rt        d t        t        |      t        |      t        |      z         D              }t!        t#        ||            }|j%                  t        t        |             |j%                  dt        |      z         t#        ||      D cg c]	  \  }}||f }}}t'        || g|i |}t)        |||||||      }|S c c}
}	w c c}}
w c c}}w c c}}w c c}}w )	a  Create a Blockwise symbolic mutable mapping

    Applies a function, ``func``, across blocks from many different input
    collections.  We arrange the pattern with which those blocks interact with
    sets of matching indices.  E.g.::

        blockwise(func, 'z', 'i', 'x', 'i', 'y', 'i')

    yield an embarrassingly parallel communication pattern and is read as

        $$ z_i = func(x_i, y_i) $$

    More complex patterns may emerge, including multiple indices::

        blockwise(func, 'z', 'ij', 'x', 'ij', 'y', 'ji')

        $$ z_{ij} = func(x_{ij}, y_{ji}) $$

    Indices missing in the output but present in the inputs results in many
    inputs being sent to one function (see examples).

    Examples
    --------
    Simple embarrassing map operation

    >>> def inc(x):
    ...    return x + 1
    >>> dict(
    ...     blockwise(
    ...         inc,
    ...         'z', 'ij',
    ...         'x', 'ij',
    ...         numblocks={'x': (2, 2)}
    ...     )
    ... )  # doctest: +NORMALIZE_WHITESPACE
    {('z', 0, 0): <Task ('z', 0, 0) inc(Alias(('x', 0, 0)))>,
    ('z', 0, 1): <Task ('z', 0, 1) inc(Alias(('x', 0, 1)))>,
    ('z', 1, 0): <Task ('z', 1, 0) inc(Alias(('x', 1, 0)))>,
    ('z', 1, 1): <Task ('z', 1, 1) inc(Alias(('x', 1, 1)))>}

    Simple operation on two datasets x and y

    >>> def add(x, y):
    ...     return x + y
    >>> dict(
    ...     blockwise(
    ...         add,
    ...         'z', 'ij',
    ...         'x', 'ij',
    ...         'y', 'ij',
    ...         numblocks={'x': (2, 2), 'y': (2, 2)}
    ...     )
    ... )  # doctest: +NORMALIZE_WHITESPACE
    {('z', 0, 0): <Task ('z', 0, 0) add(Alias(('x', 0, 0)), Alias(('y', 0, 0)))>,
    ('z', 0, 1): <Task ('z', 0, 1) add(Alias(('x', 0, 1)), Alias(('y', 0, 1)))>,
    ('z', 1, 0): <Task ('z', 1, 0) add(Alias(('x', 1, 0)), Alias(('y', 1, 0)))>,
    ('z', 1, 1): <Task ('z', 1, 1) add(Alias(('x', 1, 1)), Alias(('y', 1, 1)))>}

    Operation that flips one of the datasets

    >>> def addT(x, y):
    ...     return x + y.T
    >>> dict(
    ...     blockwise(
    ...         addT,
    ...         'z', 'ij',
    ...         'x', 'ij',
    ...         'y', 'ji',
    ...         numblocks={'x': (2, 2), 'y': (2, 2)}
    ...     )
    ... )  # doctest: +NORMALIZE_WHITESPACE
    {('z', 0, 0): <Task ('z', 0, 0) addT(Alias(('x', 0, 0)), Alias(('y', 0, 0)))>,
    ('z', 0, 1): <Task ('z', 0, 1) addT(Alias(('x', 0, 1)), Alias(('y', 1, 0)))>,
    ('z', 1, 0): <Task ('z', 1, 0) addT(Alias(('x', 1, 0)), Alias(('y', 0, 1)))>,
    ('z', 1, 1): <Task ('z', 1, 1) addT(Alias(('x', 1, 1)), Alias(('y', 1, 1)))>}

    Dot product with contraction over ``j`` index.  Yields list arguments

    >>> from dask.array.core import dotmany
    >>> dict(
    ...     blockwise(
    ...         dotmany,
    ...         'z', 'ik',
    ...         'x', 'ij',
    ...         'y', 'jk',
    ...         numblocks={'x': (2, 2), 'y': (2, 2)}
    ...     )
    ... )  # doctest: +SKIP
    {('z', 0,  0):
        <Task ('z', 0, 0) dotmany(
        List((Alias(('x', 0, 0)), Alias(('x', 0, 1)))),
        List((Alias(('y', 0, 0)), Alias(('y', 1, 0)))))
        >,
    ('z', 0,  1):
        <Task ('z', 0, 1) dotmany(
        List((Alias(('x', 0, 0)), Alias(('x', 0, 1)))),
        List((Alias(('y', 0, 1)), Alias(('y', 1, 1)))))
        >,
    ('z', 1,  0):
        <Task ('z', 1, 0) dotmany(
        List((Alias(('x', 1, 0)), Alias(('x', 1, 1)))),
        List((Alias(('y', 0, 0)), Alias(('y', 1, 0)))))
        >,
    ('z', 1,  1):
        <Task ('z', 1, 1) dotmany(
        List((Alias(('x', 1, 0)), Alias(('x', 1, 1)))),
        List((Alias(('y', 0, 1)), Alias(('y', 1, 1)))))
        >
    }

    Pass ``concatenate=True`` to concatenate arrays ahead of time.
    ``concatenate_axes`` is then called on the lists before the arguments are
    passed to the function.

    Supports Broadcasting rules

    >>> dict(
    ...     blockwise(
    ...         add,
    ...         'z', 'ij',
    ...         'x', 'ij',
    ...         'y', 'ij',
    ...         concatenate=True,
    ...         numblocks={'x': (1, 2), 'y': (2, 2)}
    ...     )
    ... )  # doctest: +SKIP
    {
    ('z', 0, 0): <Task ('z', 0, 0) add(Alias(('x', 0, 0)), Alias(('y', 0, 0)))>,
    ('z', 0, 1): <Task ('z', 0, 1) add(Alias(('x', 0, 1)), Alias(('y', 0, 1)))>,
    ('z', 1, 0): <Task ('z', 1, 0) add(Alias(('x', 0, 0)), Alias(('y', 1, 0)))>,
    ('z', 1, 1): <Task ('z', 1, 1) add(Alias(('x', 0, 1)), Alias(('y', 1, 1)))>
    }

    Include literals by indexing with ``None``

    >>> dict(
    ...     blockwise(
    ...         add,
    ...         'z', 'i',
    ...         'x', 'i',
    ...         100, None,
    ...         numblocks={'x': (2, )}
    ...     )
    ... )  # doctest: +NORMALIZE_WHITESPACE
    {('z', 0): <Task ('z', 0) add(Alias(('x', 0)), DataNode(100))>,
    ('z', 1): <Task ('z', 1) add(Alias(('x', 1)), DataNode(100))>}

    If the broadcasted value is a delayed object or other dask collection, the
    key has to be wrapped appropriately as an Alias object.

    >>> from dask import delayed
    >>> from dask._task_spec import Alias
    >>> delayed_inc = delayed(inc)(5, dask_key_name='dinc')
    >>> dict(
    ...     blockwise(
    ...         add,
    ...         'z', 'i',
    ...         'x', 'i',
    ...         Alias(delayed_inc.key), None,
    ...         numblocks={'x': (2, )}
    ...     )
    ... )  # doctest: +NORMALIZE_WHITESPACE
    {('z', 0): <Task ('z', 0) add(Alias(('x', 0)), Alias('dinc'))>,
    ('z', 1): <Task ('z', 1) add(Alias(('x', 1)), Alias('dinc'))>}

    See Also
    --------
    dask.array.blockwise
    Blockwise
       N   .r   c              3  2   K   | ]  }t        |        y wrP   )rz   rq   rx   s     r2   rs   zblockwise.<locals>.<genexpr>  s      
(W1OA(W   rP   )r*   r{   r|   )rg   rh   	enumeratesortedrz   ru   rf   appendrd   toolz	partitionmapr   rangerR   __dask_layers__r[   zipextendr   	Blockwise)funcoutputoutput_indicesr*   r{   r|   r}   arrind_pairskwargsiirx   unique_indicesrl   suba_pairs_listavalrm   inputsinputs_indicesnameindex	task_argsrr   nnew_keys
new_tokensindicesrj   subgraphs                                 r2   	blockwiser      s   j ~2H%L
 !A&&b"."Q"&NN 3<F><R2S
T2S$!Q1oa%%2SC
Tn 5s;NL!$Q$=(CCJsC01   &LA8@8HI8H1
A4%a(!+8HHI FNq,7ede$ 8 S#ouS[7I"JKLI'E<a1B1B1DA1D<HE 
(-c&k3v;X;V(W
 

 3x,-c'8,-gH56"%fn"=>"=$!Q1v"=G>3y3F3DH O] U J F ?s"   I-I-I3I9;I?8Jc                  2    e Zd ZU dZded<   ded<   ded<   ded	<   d
ed<   ded<   ded<   ded<   ded<   	 	 	 	 	 d!	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d" fdZed        Zd Zed        Z	d Z
d Zd Zd#dZd Zd Zd Z	 	 	 	 	 	 d$dZ	 d%	 	 	 	 	 	 	 d&d Z xZS )'r   a4	  Tensor Operation

    This is a lazily constructed mapping for tensor operation graphs.
    This defines a dictionary using an operation and an indexing pattern.
    It is built for many operations like elementwise, transpose, tensordot, and
    so on.  We choose to keep these as symbolic mappings rather than raw
    dictionaries because we are able to fuse them during optimization,
    sometimes resulting in much lower overhead.

    Parameters
    ----------
    output: str
        The name of the output collection.  Used in keynames
    output_indices: tuple
        The output indices, like ``('i', 'j', 'k')`` used to determine the
        structure of the block computations
    dsk: dict
        A small graph to apply per-output-block.  May include keys from the
        input indices.
    indices: tuple[tuple[str, tuple[str, ...] | None], ...]
        An ordered mapping from input key name, like ``'x'``
        to input indices, like ``('i', 'j')``
        Or includes literals, which have ``None`` for an index value.
        In place of input-key names, the first tuple element may also be a
        ``BlockwiseDep`` object.
    numblocks: Mapping[key, Sequence[int]]
        Number of blocks along each dimension for each input
    concatenate: bool
        Whether or not to pass contracted dimensions as a list of inputs or a
        single input to the block function
    new_axes: Mapping
        New index dimensions that may have been created and their size,
        e.g. ``{'j': 2, 'k': 3}``
    output_blocks: set[tuple[int, ...]]
        Specify a specific set of required output blocks. Since the graph
        will only contain the necessary tasks to generate these outputs,
        this kwarg can be used to "cull" the abstract layer (without needing
        to materialize the low-level graph).
    annotations: dict (optional)
        Layer annotations
    io_deps: dict[str, BlockwiseDep] (optional)
        Dictionary containing the mapping between "place-holder" collection
        keys and ``BlockwiseDep``-based objects.
        **WARNING**: This argument should only be used internally (for culling,
        fusion and cloning of existing Blockwise layers). Explicit use of this
        argument will be deprecated in the future.

    See Also
    --------
    dask.blockwise.blockwise
    dask.array.blockwise
    rG   r   ztuple[str, ...]r   r   rj   z8tuple[tuple[str | TaskRef, tuple[str, ...] | None], ...]r   Mapping[str, Sequence[int]]r*   bool | Noner{   zMapping[str, int]r|   set[tuple[int, ...]] | Noneoutput_blockszMapping[str, BlockwiseDep]io_depsc                   t         |   |	       || _        t        |      | _        || _        || _        t        |t              sJ g }t        |      }t        |
xs i       }
|ru|D ]p  \  }}|t        |t              rJ d       t        |t              r t        |      }||
|<   |j                  ||<   n|}|j                  ||t        |      n|f       r || _        |
xs i | _        t        |      | _        t#        | j                        | t%        fd| j                   D              rd }|| _        |xs i | _        y )Nr   zKTaskRef objects are only allowed for broadcasted inputs with None as index.c              3  >   K   | ]  \  }}||D ]  }|v  
  y wrP   r:   )rq   r   rt   rx   output_indices_sets       r2   rs   z%Blockwise.__init__.<locals>.<genexpr>B  s7      +
)	c ##  $)s   )superrT   r   rf   r   r   rj   rc   r   r[   r   r(   r   r*   r   r   r   rh   allr{   r|   )r0   r   r   rj   r   r*   r{   r|   r   r   r   _tmp_indicesdeprt   r   r   	__class__s                  @r2   rT   zBlockwise.__init__  sU    	[1#N3*	$%%% O	w}"%#S?)W  ede  c<0#C=D$'GDM&)mmIdOD##T5:c$RS $ #}"\* !!4!45"s +
!\\+
 (
 K& Br4   c                    t        | d      s0t        | j                  | j                  | j                        | _        | j
                  S )zReturns a dictionary mapping between each index specified in
        `self.indices` and the number of output blocks for that indice.
        _dims)hasattr
_make_dimsr   r*   r|   r   r;   s    r2   dimszBlockwise.dimsL  s4    
 tW%#DLL$..$--PDJzzr4   c                <    d| j                    d| j                   dS )Nz
Blockwise<z -> r@   )r   r   r;   s    r2   rC   zBlockwise.__repr__U  s    DLL>dkk]!<<r4   c                    t        | d      r| j                  d   S t        t        t        t        t        | j                                          }t        | j                  | j                  | j                  gt        t        j                  | j                              | j                  | j                   | j"                  | j$                  | j&                  | j(                  |d}d|i| _        | j                  d   S )N_cached_dictdsk)r|   r*   r{   r   r   r   keys)r   r   rf   r   rz   r   rR   r   _make_blockwise_graphrj   r   r   rg   r   concatr|   r*   r{   r   r   r   )r0   r   r   s      r2   _dictzBlockwise._dictX  s    4($$U++_eC4E.FGHD'		## ell4<<01	
 .. ,,"00YYC "'D  ''r4   c           	     @   | j                   r(| j                   D ch c]  }| j                  g| c}S t        j                  | j                  D cg c]  }t        | j                  |          c} D ch c]  }| j                  g| c}S c c}w c c}w c c}w rP   )r   r   	itertoolsr   r   r   r   )r0   prx   s      r2   get_output_keyszBlockwise.get_output_keysp  s    /3/A/AB/A!T[[%1%/ABB
 &&/3/B/BC/B!%		!%/BC
 [[1
 	
 C D
s   BB7Bc                     | j                   |   S rP   )r   )r0   keys     r2   r3   zBlockwise.__getitem__}  s    zz#r4   c                ,    t        | j                        S rP   )iterr   r;   s    r2   __iter__zBlockwise.__iter__  s    DJJr4   c                      j                   rt         j                         S t         fd j                  D              S )Nc              3  <   K   | ]  }j                   |     y wrP   r   rq   rx   r0   s     r2   rs   z$Blockwise.__len__.<locals>.<genexpr>  s     @,?qdiil,?   )r   rR   r   r   r;   s   `r2   rZ   zBlockwise.__len__  s>     !! ""#	
 @D,?,?@@	
r4   c                    t        | d      S )Nr   )r   r;   s    r2   is_materializedzBlockwise.is_materialized  s    t^,,r4   c                   d}| j                   du rddlm} t        | j                  | j
                  | j                  | j                  |      \  }}}t               }| j                  D ]:  \  }}	t        |t              r|j                  }|	%	 ||v r|j                  |       < i }
|D ]  }t               }||z   t        ||| j                        D ]x  \  }}\  }}	|	|| j                  vst!        fd|D              }|r0t#        |f|      }|j%                  t'        |             |s\|||f}b|f|z   }|j                  |       z ||z  |
| j(                  f|z   <    | j                  j+                         D ]X  \  }}|j,                  s|D ]A  }| j(                  f|z   }||   }t        |t              r|j                  }|
|xx   |hz  cc<   C Z |
S # t        $ r Y w xY w)zDetermine the necessary dependencies to produce `output_blocks`.

        This method does not require graph materialization.
        NTr   concatenate_axesc              3  (   K   | ]	  }|     y wrP   r:   rq   rr   coordss     r2   rs   z/Blockwise._cull_dependencies.<locals>.<genexpr>  s     &?$Qvay$   )r{   dask.array.corer   _get_coord_mappingr   r   r*   r   rh   rc   r   r   addri   r   r   rf   _lol_productupdater   r   rd   r<   )r0   all_hlg_keysr   r{   
coord_mapsconcat_axesdummies
const_depsargrt   key_deps
out_coordsdepscmapaxes
arg_coordstupsr   io_depvalid_key_depr   s                       @r2   _cull_dependencieszBlockwise._cull_dependencies  s    t#G .@IINNLL.
*[' U
HC#w'gg{l*"s+ % 'J5D')F*-j+t||*T&dJS#?s$,,'>!&&?$&?!?J+SFJ?GDM2&$/t#<D #v
2 +U 59:4EHdkk^j01 ("  <<--/KC##"/J;;.:5C$*:$6M!-9(5(9(9SMm_4M #0 0 ? ! s   G	G*)G*c                    t        | j                  | j                  | j                  | j                  | j
                  | j                  | j                  || j                  | j                  
      S )N)r{   r|   r   r   r   )
r   r   r   rj   r   r*   r{   r|   r   r   )r0   r   s     r2   _cullzBlockwise._cull  sW    KKIILLNN((]]'((LL
 	
r4   c                "    t               }|D ])  }|d    j                  k(  s|j                  |dd         +  j                  ||      } fd j                  D        }t        |      t        |      k7  r j                  |      }||fS  |fS )Nr   r   c              3  <   K   | ]  }j                   |     y wrP   r   r   s     r2   rs   z!Blockwise.cull.<locals>.<genexpr>  s     C/B!1/Br   )rh   r   r   r   r   r   rR   r   )r0   r   r   r   r   culled_depsout_size_iterculled_layers   `       r2   cullzBlockwise.cull  s     /2eC1v$!!#ab'*  --lMJCt/B/BC#k"22::m4L,,$$r4   c                   |D ch c]  }t        |       }}dt        j                  v rZ| j                         |z
  rJ | j                  j                         D ](  \  }}||v st        d |D         D ]  }|g||v rJ  * d}	g }
| j                  D ]u  \  }}t        |      r||v rd}	t        ||      }n?t        |t              r/|j                  |v r!d}	t        t        |j                  |            }|
j                  ||f       w i }| j                  j                         D ]  \  }}||v rd}	t        ||      }|||<    ||	rddlm} t        |t               sJ t#        t        | j$                  j                  |      |j&                  | j$                  t        t)        t+        |
                        }|
j                  t        |      d f       n;| j$                  j-                  i t        | j$                  j                  |            }t/        t        | j0                  |      | j2                  ||
|| j4                  | j6                  | j8                  | j:                  | j<                  
      |d uxr |	fS c c}w )	NPYTEST_CURRENT_TESTc              3  D   K   | ]  }t        t        |              y wrP   )rg   r   )rq   nbis     r2   rs   z"Blockwise.clone.<locals>.<genexpr>  s     *Jr4c
+;rs    TFr   )chunksr   )
r   r   rj   r   r*   r{   r|   r   r   r   )r   osenvironr   r*   rd   r   r   r   r   rc   r   r   r   dask.graph_manipulationr   rG   r   rj   bindrz   rR   
substituter   r   r   r{   r|   r   r   r   )r0   r   seedbind_torl   namesr   nbblockis_leafr   idxvr*   nbvr   newtasks                   r2   clonezBlockwise.clone  sZ    044t!"1%t4 !BJJ.++-444 NN002b5=!(*Jr*J!K $~u~555 "L 3
 ||GAt !}ea&Aw'AEEUNIaeeT23NNAt9% $ /1	nn**,FAsEza&IaL	 - 76 gs+++$))--.		G56	G NNGG,d34ii**29TYY]]D3Q*RG  d3#22# ,,"00 ,, D ,W
 	
k 5s   J)NNNNN)r   rG   r   zIterable[str]rj   r   r   zCIterable[tuple[str | TaskRef | BlockwiseDep, Iterable[str] | None]]r*   r   r{   r   r|   zMapping[str, int] | Noner   r   r   zMapping[str, Any] | Noner   z!Mapping[str, BlockwiseDep] | Noner\   )r   rh   r   r   rE   z$tuple[Layer, Mapping[Key, set[Key]]]rP   )r   zset[Key]r   r   r  z
Key | NonerE   ztuple[Layer, bool])rB   rH   rI   rJ   rK   rT   rL   r   rC   r   r   r3   r   rZ   r   r   r   r   r	  __classcell__)r   s   @r2   r   r     s\   3j K##
JEE**..'' $(-15904598'8' &8' 	8'
 U8' /8' !8' +8' 38' .8' 38't  = ( (.
 
-<|
%%'/%	-%. #	I
I
 I
 	I

 
I
r4   r   c           	        t               }t               }|D ]1  \  }}|	|j                  |       |D ]  }	|j                  |	        3 t        |      |k(  s	J ||f       |t        |      z
  }
i i }}t        |      D ]  \  }}|||<   d||<    g }t        |
      D ]e  \  }}d|z  t        |      z   ||<   d|z  dz   t        |      z   ||<   |rdn| |   }|j	                  t        t        | |               dg|z  g       g t        t        j                  j                  |            }|dz  }g }g }|D ]  \  }}|q|j	                  t        |||         D cg c]  \  }}|dk(  r||   n||    c}}       |j	                  t        |      D cg c]  \  }}||
v s| c}}       y|j	                  d       |j	                  d        |||fS c c}}w c c}}w )a  Calculate coordinate mapping for graph construction.

    This function handles the high-level logic behind Blockwise graph
    construction. The output is a tuple containing: The mapping between
    input and output block coordinates (`coord_maps`), the axes along
    which to concatenate for each input (`concat_axes`), and the dummy
    indices needed for broadcasting (`dummies`).

    Used by `make_blockwise_graph` and `Blockwise._cull_dependencies`.

    Parameters
    ----------
    dims : dict
        Mapping between each index specified in `argpairs` and
        the number of output blocks for that index. Corresponds
        to the Blockwise `dims` attribute.
    out_indices : tuple
        Corresponds to the Blockwise `output_indices` attribute.
    numblocks : dict
        Corresponds to the Blockwise `numblocks` attribute.
    argpairs : tuple
        Corresponds to the Blockwise `indices` attribute.
    concatenate : bool
        Corresponds to the Blockwise `concatenate` attribute.
    Nr   r   r   )r   )rh   r   r   rR   r   rg   r   rf   r   chainfrom_iterabler   )r   out_indicesr*   argpairsr{   block_namesall_indicesr   rt   rn   dummy_indices	index_poszero_posrx   _dummies_listrepsr   r   r   r   r  r   s                         r2   r   r   9  s   B %K%K	c?OOD!"  
 y>[(B9k*BB(#k"22M bxIK(3	# ) MM*3Q[!11	#A	C$44qT#Yd5c#34qcDjAB	 + IOO11-@AGtOG J KS? "%S)C.!9!92 $&7HQK	!<!9 inSnda]@RnSTd#t$  {G++
  Ts   GG!%G!)r*   r{   r|   r   r   r   r   c               "   |t        d      |xs i }|xs i }t        t        j                  d|
            }|du rddlm} |xs t        |||      }t        |||||      \  }}}|xs5 t        t        j                  |D cg c]  }t        ||          c}       }ddlm} i }|D ][  }| }||z   g }t        ||||	d      D ]  \  }}\  }}}|| j                  vr|Et        |t         t"        f      s|j%                  | |d |      i      }n|j%                  ||i      }bt'        fd|D              }||v rKt)        ||   j+                  ||            }t        |t               s	 |d |      }|j%                  ||i      }i }|r%t-        |f|d	      }|rt/        ||||      }|||<   n	|g|||<   |j%                  |      }
 |f|z   }t        |t.              sJ t/        j0                  |g|d
|i||<   ^ |S c c}w )Nz$Missing required numblocks argument.r   Tr   r   )DataNode)strictc              3  (   K   | ]	  }|     y wrP   r:   r   s     r2   rs   z(_make_blockwise_graph.<locals>.<genexpr>  s     7$Qvay$r   
as_taskrefr   )
ValueErrorrg   r   r   r   r   r   r   r   r   r   dask._task_specr  r   r}   rc   r   r   r   rf   r   r8   r   r   r"   ) rj   r   r  r*   r{   r|   r   r   r   r   r   r  r   r   r   rx   r  r   r   	this_taskargsr   r   r   rt   r   r   r   re   r   new_keyr   s                                   @r2   r   r     sa    ?@@~2HmGEOOA|45HdC <:h	8<D *<*&Zg " TK@KqE$q'NK@A&M )
C#
	g%+.XtD,
 ,
'D$
cC $+++{!#	7';< ) 4 4c8D#;N5O PI ) 4 4c3Z @I7$77Jg~!'#,"2"2:z"JK!#y1"4-C%00#s<	'
tLD"#CdDA $DI!$ 2z 2DI%006	7,
8 )j()T***yy?T?w?GE $F JQ As   Hc                     srt               S  S t        d   t              r1rt         fdd   D         S t         fdd   D              S t	         d   fz   dd       S )aJ  List of list of tuple keys, similar to `itertools.product`.

    Parameters
    ----------
    head : tuple
        Prefix prepended to all results.
    values : sequence
        Mix of singletons and lists. Each list is substituted with every
        possible value and introduces another level of list in the output.

    Examples
    --------
    >>> _lol_product(('x',), (1, 2, 3))
    ('x', 1, 2, 3)
    >>> _lol_product(('x',), (1, [2, 3], 4, [5, 6]))  # doctest: +NORMALIZE_WHITESPACE
    [[('x', 1, 2, 4, 5), ('x', 1, 2, 4, 6)],
     [('x', 1, 3, 4, 5), ('x', 1, 3, 4, 6)]]
    r   c              3  H   K   | ]  }t        |fz   d d         ywr   Nr  r   rq   rn   r  headvaluess     r2   rs   z_lol_product.<locals>.<genexpr>  s1      & !fQRjZPP&   "c              3  H   K   | ]  }t        |fz   d d         ywr%  r&  r'  s     r2   rs   z_lol_product.<locals>.<genexpr>
  s1      "A TQD[&*LL"r*  r   Nr  )r   rc   rg   r   r   )r(  r)  r  s   ```r2   r   r     s    & 4= 	F1It	$#AY     
 DF1I</
SSr4   c           	         |s| S |d   |vrt        | ||d      fz   |dd ||      S ||d      D cg c]  }t        | |fz   |dd ||       c}S c c}w )a  List of list of tuple keys

    Parameters
    ----------
    head : tuple
        The known tuple so far
    ind : Iterable
        An iterable of indices not yet covered
    values : dict
        Known values for non-dummy indices
    dummies : dict
        Ranges of values for dummy indices

    Examples
    --------
    >>> lol_tuples(('x',), 'ij', {'i': 1, 'j': 0}, {})
    ('x', 1, 0)

    >>> lol_tuples(('x',), 'ij', {'i': 1}, {'j': range(3)})
    [('x', 1, 0), ('x', 1, 1), ('x', 1, 2)]

    >>> lol_tuples(('x',), 'ijk', {'i': 1}, {'j': [0, 1, 2], 'k': [0, 1]}) # doctest: +NORMALIZE_WHITESPACE
    [[('x', 1, 0, 0), ('x', 1, 0, 1)],
     [('x', 1, 1, 0), ('x', 1, 1, 1)],
     [('x', 1, 2, 0), ('x', 1, 2, 1)]]
    r   r   N)
lol_tuples)r(  rt   r)  r   rm   s        r2   r-  r-    s    6 
1vW$&Q.!22CGVWMM HOsSTv
GV!Jtqd{CGVW=
 	
 
s   Ac                    t        | |      }|j                  | j                  k7  r)|} t        | |      }|j                  | j                  k7  r)|S )a  High level optimization of stacked Blockwise layers

    For operations that have multiple Blockwise operations one after the other, like
    ``x.T + 123`` we can fuse these into a single Blockwise operation.  This happens
    before any actual tasks are generated, and so can reduce overhead.

    This finds groups of Blockwise operations that can be safely fused, and then
    passes them to ``rewrite_blockwise`` for rewriting.

    Parameters
    ----------
    graph : HighLevelGraph
    keys : Iterable
        The keys of all outputs of all collections.
        Used to make sure that we don't fuse a layer needed by an output

    Returns
    -------
    HighLevelGraph

    See Also
    --------
    rewrite_blockwise
    )r   )_optimize_blockwiser}   )graphr   outs      r2   optimize_blockwiser2  7  sR    2 e$
/C


e00
0!%d3 

e00
0 Jr4   c           	     F   |D ch c]  }t        |      t        u r|d   n| }}| j                  }t        | j                        }| j                  D ch c]  }|j                  |      r| }}t        |      }i }i }	t               }
t               }|rc|j                         }||
v s||vr|
j                  |       t        ||   t              r|h}t        |      }|||   j                  j                         z  }|r|j                         |vr|j                         )t        |   t              s|j                         N|k7  r|v r|j                         i|   j                  ||   j                  k7  r|j                         t!        fd||   j"                  D              dkD  r|j                         |rKt%        |t'        t)        |               j*                  |   j*                        s|j                         |j                         t        |   j,                        }|   j"                  D ch c]  \  }}|s	|D ]  }|  }}}}|j/                  |      }| j                  j                  d      D ]8  }|r#t1        ||         dk  r|j                  |       (|j                  |       : |rt3        |D cg c]  }||   	 c}      }|||<   t               }|D ]=  }|t        | j                  |   D ch c]  }||vr|| j                  v r| c}      z  }? |j"                  D ]8  \  }}||t5        | j                  |g      z  }#||vs(|j                  |       : ||	|<   nZ||   ||<   | j                  j                  |t                     |	|<   |j7                  | j                  j                  |d             |rct9        ||	      S c c}w c c}w c c}}}w c c}w c c}w )Nr   c              3  4   K   | ]  \  }}|	|k(    y wrP   r:   )rq   rl   rt   r   s      r2   rs   z&_optimize_blockwise.<locals>.<genexpr>~  s      X/DVQS/Ds   

r   r:   )rA   rf   layersr   r}   r8   rg   rh   popr   rc   r   r   r   r   r{   sumr   _can_fuse_annotationsnextr   r   r   
issupersetrR   rewrite_blockwiser   r   r    )
full_graphr   rl   keepr5  
dependentsrootsstackr1  r}   seenio_nameslayerblockwise_layersr   r   _rt   rx   input_indicesis_io_supersetdl	new_layernew_depsrm   r   s                             @r2   r/  r/  W  s   59:TDGu$AaD!+TD:Fj556J"))C)11BQ)ECKE
CL5DuH
		D=E/ fUmY/ %w'(Du--2244Hhhjf$LL%!&+y9LL%%<C4KLL%#;**fUm.G.GGLL%Xve}/D/DXX LL%#,A4%5 678DD3K++- LL% !$$S) "%VC[%?%?!@$*3K$7$7!$7&!S31AA$7  ! "0!:!:=!I#0044S"=A%#jm*<*AQ >K Z *>N*O>N6!9>N*OPI"CJ
 uH%C ",!8!8!;!;A$44j>U>U9U !;  & "))19j.E.Es KKHh&LLO	 *
 #+LCJ","9"9"="=eSU"KLLL0044UB?@a d #|,,{ ; Dh! +Ps)   PP-P
P)P-P+Pc                P    | dz   dj                  d t        |      D              z   S )NrE  c              3  2   K   | ]  }t        |        y wrP   rG   r   s     r2   rs   z_unique_dep.<locals>.<genexpr>  s     :	1A	r   )joinrg   )r   rt   s     r2   _unique_deprP    s$    9sxx:S	::::r4   c                    | |k(  ryt         j                  j                  d      du ryh d| rt        fd| D              r|rt        fd|D              ryy)z
    Treat the special annotation keys, as fusable since we can apply simple
    rules to capture their intent in a fused layer.
    Tzoptimization.annotations.fuseF>   retriesworkerspriority	resourcesallow_other_workersc              3  &   K   | ]  }|v  
 y wrP   r:   rq   rl   fusables     r2   rs   z(_can_fuse_annotations.<locals>.<genexpr>       -1aQ'\1   c              3  &   K   | ]  }|v  
 y wrP   r:   rX  s     r2   rs   z(_can_fuse_annotations.<locals>.<genexpr>  rZ  r[  )daskconfigr8   r   )r   brY  s     @r2   r8  r8    sS    
 	Av{{675@TG-1---1--r4   c            
     X   t        j                  |  }| D cg c]  }d|v s|d    }}|rt        |      |d<   | D cg c]  }d|v s|d    }}|rt        |      |d<   | D cg c]  }d|v s|d    }}|rt        j                  t        g| |d<   | D cg c]  }d|v s|d    }}|r5t	        t        j                  |D cg c]  }t        |       c}       |d<   | D cg c]  }d|v s|d    }}|rt        |      |d<   |S c c}w c c}w c c}w c c}w c c}w c c}w )zf
    Given an iterable of annotations dictionaries, fuse them according
    to some simple rules.
    rR  rT  rU  rS  rV  )r   mergemax
merge_withrg   rh   intersectionr   )	r!  r   r   rR  
prioritiesrU  rS  wrV  s	            r2   _fuse_annotationsrg    s`    ++t$K%)<TY!^q|TG<!$WI)-AAq!J-JA"%j/J)-BA1A;IB#(#3#3C#D)#DK %)<TY!^q|TG<!%c&6&68QAQ8Q&R!SI +/*.Q2G12L
 $   -01D-E)*- = B C =8Qs?   	DD	DD%	D/D	D%DD")	D'3D'c                p   t        |       dk(  r| d   S t        | D cg c]  }|j                  s|j                   c} }| D ci c]  }|j                  | } }| j	                         D ci c]4  }|j                  |j
                  D ch c]  \  }}|	|| v s| c}}6 }}}}t        |      }d t        j                         D        }|j                         D 	cg c]
  \  }	}|r	|	 c}}	\  }
t        | |
   j
                        }| |
   j                  }| |
   j                  }| |
   j                  }|j                  |i}d}|rd}t        |      D ]  \  }\  }}||| vrd}||
k(  r|nt!        ||      }|j                         D 	ci c]"  \  }	}|	|j#                  t%        |      |i      $ }}	}|j'                  |       t)        |dz   t        |      dz         D ci c]  }t%        |      t%        |dz
         }}|j                         D 	ci c]  \  }	}|	|j#                  |       }}	}| |   j
                  }t+        t-        | |   j.                  |            }|D ch c]!  \  }}||D ]  }|| |   j.                  vr| # }}}}t+        t-        ||            }|j1                  |       |D cg c]  \  }}|t3        ||      f }}}| |   j                  j                         D ]  \  }	}||||	   <    i }t        |      D 	ci c]  \  }\  }	}t5        |	      |f| }}	}}t        |      D ]v  \  }}t5        |d         |d   f} | |v rt%        ||          |t%        |      <   8t        |      || <   t%        t        |            |t%        |      <   |j7                  |       x ||k7  r|}!n|}!|!| |   j                  j#                  ||!      i}"|j1                  |"        |rg }i }#i }t        |      D ]I  \  }}|d   ||#v r	|#|   ||<   |d   t        |      |#|<   t        |      ||<   |j7                  |       K |j                         D 	ci c]  \  }	}t%        |	      t%        |       }}	}|j                         D 	ci c]*  \  }	}|	|j9                         vs|	|j#                  |      , }}	}t;        j<                  |j	                         d|
i}|D 	ch c]
  \  }	}|	|	 }$}	}t?        j@                  | j	                         D cg c]  }|jB                   c}      }%|%j                         D 	ci c]  \  }	}||	|$v s|	| }%}	}i }&| j	                         D ]  }|&j1                  |jD                          tG        |
| |
   j.                  |||%||||&	      S c c}w c c}w c c}}w c c}}}w c c}}	w c c}}	w c c}w c c}}	w c c}}}w c c}}w c c}}	}w c c}}	w c c}}	w c c}}	w c c}w c c}}	w )	a  Rewrite a stack of Blockwise expressions into a single blockwise expression

    Given a set of Blockwise layers, combine them into a single layer.  The provided
    layers are expected to fit well together.  That job is handled by
    ``optimize_blockwise``

    Parameters
    ----------
    inputs : list[Blockwise]

    Returns
    -------
    blockwise: Blockwise

    See Also
    --------
    optimize_blockwise
    r   r   c              3  N   K   | ]  }d D ]  }||rt        |      ndz      yw)ABCDEFGHIJKLMNOPQRSTUVWXYZ NrN  )rq   rH  rr   s      r2   rs   z$rewrite_blockwise.<locals>.<genexpr>  s2      "A-A 	
qSVb!- 	""s   #%TFr   r   )r*   r|   r{   r   r   )$rR   rg  r   r   r)  r   r   r   countrd   rg   r|   r{   rj   r   r   rP  r   rz   r6  r   r[   r   r   r   ru   idr   r   r   r"   r   ra  r*   r   r   )'r   rx   fused_annotationsinprH  rm   r}   r>  new_index_iterrl   rootr   r|   r{   rj   r   changedr   current_dep_indices	local_depr   new_indicesrE  jrn   
contractedextrar   inds	index_mapr   r   id_keyr   new_dskrA  indices_checkr*   r   s'                                          r2   r;  r;    sg   & 6{aay)!'	9A1==!--	9 *00#cjj#oF0 ==?"C 	

3;;P;41a!-AKQ;PP"   l+J"N ',,.8.DAqaa.8FT 6$<''(Gd|$$H,**K$<D88T
CG
-6w-?)A)("*& G  #d{CAT0UI
 NQYY[MXTQ1<<!3Y ?@@[   KKN q1uc'lQ&677A  "OAE$::7   58IIK@KDAq1all3''KC@ !+--Ks6#;557JKLC ('DAq=AF3K666	   '   Z89EJJu?JK{tq!Az!S12{KK s,,2241#$Q  5 C>G>PQ>Pla!T"Q%)>PIQ&{3	EU1X,a1Y&/>y?P/QC+,(+GIf%/>s7|/LC+,NN5) 4 iF3K,,777EFG JJwI .@ T KD
C'"1Q4T	!WCFtk*Q%CFq! # ?Biik
Jkda?1q11kC
J,/IIK
OKDAq1CHHJ;N1all3KC
O99cjjl--D#*<741aamQ7M<fmmoFosS]]oFGI"+//"3W"3$!QqyADVA"3IW G]]_qyy!  t##%
 
w 
:0P 98 A L R@ K
O<FWs   WWW/W)
W#W#W#"W)%
W00W0'W61W<%X;&X
X)X
 X5X!X!
X'X'X-/X2>X2#W)c                &    fd}t        ||       S )a  
    >>> lol = [('x', 1, 0), ('x', 1, 1), ('x', 1, 2)]
    >>> nblocks = (4, 1, 2)  # note singleton dimension in second place
    >>> lol = [[('x', 1, 0, 0), ('x', 1, 0, 1)],
    ...        [('x', 1, 1, 0), ('x', 1, 1, 1)],
    ...        [('x', 1, 2, 0), ('x', 1, 2, 1)]]

    >>> zero_broadcast_dimensions(lol, nblocks)  # doctest: +SKIP
    [[('x', 1, 0, 0), ('x', 1, 0, 1)],
     [('x', 1, 0, 0), ('x', 1, 0, 1)],
     [('x', 1, 0, 0), ('x', 1, 0, 1)]]

    See Also
    --------
    lol_tuples
    c           	     P    | d   ft        d t        | dd        D              z   S )Nr   c              3  4   K   | ]  \  }}|d k(  rdn|  yw)r   r   Nr:   )rq   rx   rH  s      r2   rs   z>zero_broadcast_dimensions.<locals>.<lambda>.<locals>.<genexpr>  s"     !UATAqAv!1"4ATs   r   )rf   r   )tnblockss    r2   <lambda>z+zero_broadcast_dimensions.<locals>.<lambda>  s(    1Q4'E!UQqrUGAT!UUUr4   )r&   )lolr  fs    ` r2   zero_broadcast_dimensionsr    s    $ 	VAq#&&r4   c                v   | D cg c]  \  }}|	||f }}}t        j                  t        j                  t         j                  |t         j                  |j	                               D 	cg c]  \  \  }}\  }}	t        ||	       c}	}}      }
t        j                  d|
      }|j	                         D ci c]  \  }}||D ch c]  \  }}|	 c}} }}}}}|j	                         D ci c]$  \  }}|t        |      dkD  r|t        |      z
  n|& }}}|rt        j                  ||      S |r9t        t        t        |j                                     dhk(  st        d|z        t        j                  t         j                  |      S c c}}w c c}	}}w c c}}w c c}}}}w c c}}w )a  Find block dimensions from arguments

    Parameters
    ----------
    argpairs : iterable
        name, ijk index pairs
    numblocks : dict
        maps {name: number of blocks}
    sentinels : iterable (optional)
        values for singleton dimensions
    consolidate : func (optional)
        use this to reduce each set of common blocks into a smaller set

    Examples
    --------
    >>> argpairs = [('x', 'ij'), ('y', 'ji')]
    >>> numblocks = {'x': (2, 3), 'y': (3, 2)}
    >>> broadcast_dimensions(argpairs, numblocks)
    {'i': 2, 'j': 3}

    Supports numpy broadcasting rules

    >>> argpairs = [('x', 'ij'), ('y', 'ij')]
    >>> numblocks = {'x': (2, 1), 'y': (1, 3)}
    >>> broadcast_dimensions(argpairs, numblocks)
    {'i': 2, 'j': 3}

    Works in other contexts too

    >>> argpairs = [('x', 'ij'), ('y', 'ij')]
    >>> d = {'x': ('Hello', 1), 'y': (1, (2, 3))}
    >>> broadcast_dimensions(argpairs, d)
    {'i': 'Hello', 'j': (2, 3)}
    r   r   zShapes do not align %s)r   r   rO  firstrd   r   groupbyrR   rh   valmapr   r)  r  )r  r*   	sentinelsconsolidater   rt   	argpairs2rn   ry  r   Lgrl   rm   rx   rH  g2s                    r2   broadcast_dimensionsr    sv   H )1DfaCO!SID ).

YY__5F)	
)$	D9At dO)	
	A 	aA*+'')4)$!Q1141aQ1	)A4AB	KA!3q6A:QY1
4B	K||K,,	#c#ryy{+,31A566<<R((+ E	
 4	Ks.   
FF/F ?F-
F'F-
7)F5'F-
c                    t        | |      }|j                         D ]%  \  }}t        |t              rt	        |      nd||<   ' |S )zReturns a dictionary mapping between each index specified in
    `indices` and the number of output blocks for that indice.
    r   )r  rd   rc   rf   rR   )r   r*   r|   r   rl   rm   s         r2   r   r     sB      3D 1&q%0#a&aQ !Kr4   c           	        	
 t         j                  d      }t         j                  d      t              	 j                  j	                         D ]  \  }
 j                  |   }t        
t              s&t        |      dkD  s5t        fd|D              rJt        	fd|D              s_t         
fd|D              sut        j                  
g|D cg c]  }||   	 c} }t        ||t        |            \  }}|D ]  }||= |= 
 |||<   t               |<    t        |      S c c}w )a  
    Fuse nearby layers if they don't have dependencies

    Often Blockwise sections of the graph fill out all of the computation
    except for the initial data access or data loading layers::

      Large Blockwise Layer
        |       |       |
        X       Y       Z

    This can be troublesome because X, Y, and Z tasks may be executed on
    different machines, and then require communication to move around.

    This optimization identifies this situation, lowers all of the graphs to
    concrete dicts, and then calls ``fuse`` on them, with a width equal to the
    number of layers like X, Y, and Z.

    This is currently used within array and dataframe optimizations.

    Parameters
    ----------
    graph : HighLevelGraph
        The full graph of the computation
    keys : list
        The output keys of the computation, to be passed on to fuse

    See Also
    --------
    Blockwise
    fuse
    T)copyr   c              3  (   K   | ]	  }|     y wrP   r:   )rq   r   r}   s     r2   rs   zfuse_roots.<locals>.<genexpr>  s     :TcS)Tr   c              3  @   K   | ]  }t        |         d k(    yw)r   N)rR   )rq   r   r>  s     r2   rs   zfuse_roots.<locals>.<genexpr>  s      >#C
3(A-s   c              3  j   K   | ]*  }j                   j                  |   j                   k(   , y wrP   )r   r5  )rq   r   r0  rC  s     r2   rs   zfuse_roots.<locals>.<genexpr>  s.     WRV3E%%c):)F)FFRVs   03)	ave_width)r%   r5  r}   r   rd   rc   r   rR   anyr   r   ra  r"   rh   r    )r0  r   r5  r   r   r   newrE  r}   r>  rC  s   `       @@@r2   
fuse_rootsr    s   @ D1Fu11=Ll+J||))+e!!$'ui(D	A:T::>>>WRVWW++eDd&Cdsvc{d&CDC#ts4y9FC3K %  F4L!$L# ,& &,// 'Ds   "D;c           
        t        | t              rt        | j                        | ffS t	        j
                  |       r?t        | d      r| j                         } t        |       }t        |j                        |ffS t        |       t        t        t                           u rt        |       } net        |       t        t        t                           u rt        |       } n2t        |       t        t        t                           u rt        |       } t        |       }|t        t        t        fv r~t        j                  d | D        d      \  }}t        t!        j"                  t!        j$                  |      t&                    }|s| dfS t)        | }|t        urt+        d ||      }||fS |t,        u rDt/        | j1                         D cg c]	  \  }}||g c}}      \  }}|s| dfS t3        |      |fS |t4        u rHt/        | j6                  | j8                  | j:                  g      \  }}|s| dfS t+        d t4        g| |fS t=        |       rt/        t?        |       D cg c]:  }t        | |j@                        r"|j@                  tC        | |j@                        g< c}      \  }}|s| dfS 	 t?        |       D ci c]9  }t        | |j@                        r!|j@                  tC        | |j@                        ; }}tE        | fi | t+        d |fi t-        |      |fS t        jL                  |       r1t/        | D cg c]  }| c}      \  }}|s| dfS t+        d |g| |fS | dfS c c}}w c c}w c c}w # tF        tH        f$ rB}	t        |	tH              sdtK        |	      v rtI        d| d      |	tG        d| d	      |	d }	~	ww xY wc c}w )
Noptimizec              3  2   K   | ]  }t        |        y wrP   )'_blockwise_unpack_collections_task_spec)rq   es     r2   rs   z:_blockwise_unpack_collections_task_spec.<locals>.<genexpr>E  s     FA4Q7r   r   r   r:   zis declared with init=FalsezFailed to unpack zF instance. Note that using fields with `init=False` are not supported.z> instance. Note that using a custom __init__ is not supported.)'rc   r   r   _keyr   is_dask_collectionr   r  r   rA   r   rg   rf   rh   r   unzipr   uniquer   rm  r   r   r[   r  rd   r   slicestartstopstepr
   r	   r   getattrr   ri   r  rG   is_namedtuple_instance)
expr	finalizedtypr!  collectionsrl   rm   r  _fieldsr  s
             r2   r  r  )  s    $ tyy!D7**t$4$==?DTN	y~~&44DzT$tv,''Dz	dtDM*	*T{	dtDK(	(4y
t*C
tUC  !KKFF
k ELLk)BKL8OT{d?c4(D[  
d{C $

-1aV-
k 8ODz;&&
e|CZZDII.
k 8OD%'$'44DC  %A4( qvv./%
k 8O	  %A4( aff--%  
 D$G$ D#,d,k99##D)CPTDUPT1QPTDUVk8OD#%%{228Oe .  :& 
	!Z(,ISQRV,S 'u -R R 
  'u -J J 
	 EVs<   M5
*?M;:N >N N 		O N O=OO)F)r:   )r   dict | Noner_  r  rE   r+   )r!  r[   rE   r[   ))r   )r   N)r0  r    r   rg   )M
__future__r   r   r   collections.abcr   r   r   r   dataclassesr	   r
   r   r   mathr   typingr   tlzr   r]  r   r   r  r   r   r   r   r   r   	dask.baser   r   r   	dask.corer   r   r   r   dask.delayedr   r   dask.highlevelgraphr    r!   dask.optimizationr"   dask.typingr#   
dask.utilsr$   r%   r&   r(   rN   r_   re   ru   _BLOCKWISE_DEFAULT_PREFIXrz   r   r   r   r   r   r-  r2  r/  rP  r8  rg  r;  r  r  r   r  r  r:   r4   r2   <module>r     s9   "  	 A A 5 5       M M < < F F * 5 "  D D8; 8;v]!| ]!@ & : 1  8  iXd
 d
NX,@ 		Rj&TR"
J@^-B;
(D]@ ' '*9)x70t[r4   