
    Cdwm                       d Z ddlmZ ddlZddlZddlmZ ddlmZm	Z	m
Z
mZ ddlmZmZmZ ddlmZ ddlZddlZddlmZ dd	lmZ dd
lmZ ddlmZmZmZ ddlm Z  ddl!m"Z" ddl#m$Z$ ddl%m&Z& ddl'm(Z( d Z)d Z*d Z+d Z,d Z-	 	 	 	 	 d,dZ.d Z/d Z0d Z1d Z2d Z3d Z4d Z5	 d-d Z6d! Z7 G d" d#          Z8d$ Z9d% Z:d& Z;d' Z<d.d+Z=dS )/z
The rechunk module defines:
    intersect_chunks: a function for
        converting chunks to new dimensions
    rechunk: a function to convert the blocks
        of an existing dask array to new chunks or blockshape
    )annotationsN)reduce)chaincompresscountproduct)add
itemgettermul)warn)
accumulate)config)getitem)Arrayconcatenate3normalize_chunks)validate_axis)empty)tokenize)HighLevelGraph)parse_bytesc                     fd| D             S )zInternal utility for cumulative sum with label.

    >>> cumdims_label(((5, 3, 3), (2, 2, 1)), 'n')  # doctest: +NORMALIZE_WHITESPACE
    [(('n', 0), ('n', 5), ('n', 8), ('n', 11)),
     (('n', 0), ('n', 2), ('n', 4), ('n', 5))]
    c                    g | ]I}t          t          fd t          |          z   z  t          t          d|z                                 JS )   r   )tupleziplenr   r	   ).0bdsconsts     2lib/python3.11/site-packages/dask/array/rechunk.py
<listcomp>z!cumdims_label.<locals>.<listcomp>&   sZ        	c5(a#c((l+ZTCZ-H-HIIJJ       )chunksr!   s    `r"   cumdims_labelr'      s.          r$   c                ^    t          t          | |z   t          d                              S )aG  

    >>> new = cumdims_label(((2, 3), (2, 2, 1)), 'n')
    >>> old = cumdims_label(((2, 2, 1), (5,)), 'o')

    >>> _breakpoints(new[0], old[0])
    (('n', 0), ('o', 0), ('n', 2), ('o', 2), ('o', 4), ('n', 5), ('o', 5))
    >>> _breakpoints(new[1], old[1])
    (('n', 0), ('o', 0), ('n', 2), ('n', 4), ('n', 5), ('o', 5))
    r   key)r   sortedr
   )cumoldcumnews     r"   _breakpointsr.   ,   s)     Z]];;;<<<r$   c                x   d | D             }t          |          dz
  }|d         d         }d}d}d}d}g }g }	t          dt          |                     D ]}
| |
         \  }}| |
dz
           \  }}|dk    r|}|	r|                    |	           g }	nd}||z
  |z   }|}||k    rI|dk    r|dz  }|}|dk    r5|dk    r/||k    r(t          ||          }|	                    ||f           n|	                    |t          ||          f           |dk    r	|dz  }d}|}|	r|                    |	           |S )a#  
    Internal utility to intersect chunks for 1d after preprocessing.

    >>> new = cumdims_label(((2, 3), (2, 2, 1)), 'n')
    >>> old = cumdims_label(((2, 2, 1), (5,)), 'o')

    >>> _intersect_1d(_breakpoints(old[0], new[0]))  # doctest: +NORMALIZE_WHITESPACE
    [[(0, slice(0, 2, None))],
     [(1, slice(0, 2, None)), (2, slice(0, 1, None))]]
    >>> _intersect_1d(_breakpoints(old[1], new[1]))  # doctest: +NORMALIZE_WHITESPACE
    [[(0, slice(0, 2, None))],
     [(0, slice(2, 4, None))],
     [(0, slice(4, 5, None))]]

    Parameters
    ----------

    breaks: list of tuples
        Each tuple is ('o', 8) or ('n', 8)
        These are pairs of 'o' old or new 'n'
        indicator with a corresponding cumulative sum,
        or breakpoint (a position along the chunking axis).
        The list of pairs is already ordered by breakpoint.
        Note that an 'o' pair always occurs BEFORE
        an 'n' pair if both share the same breakpoint.
    Uses 'o' and 'n' to make new tuples of slices for
    the new block crosswalk to old blocks.
    c                *    g | ]}|d          dk    |S )r   or%   )r   pairs     r"   r#   z!_intersect_1d.<locals>.<listcomp>e   s!    999$q'S..t...r$      r   r   nr1   )r   rangeappendslice)breakso_pairslast_old_chunk_idx	last_o_brstartlast_endold_idx
last_o_endretret_nextidxlabelbr
last_labellast_brendslcs                    r"   _intersect_1drJ   :   s   V :9999GW)AIEHGJ
CHQF$$ & & 3K	r$S1Wo
G E 

8$$$E7lU"==
 ||1 
||
c 1 1??  
J77COO%7$=>>> # %s"3"34555C<<qLGEJ 

8Jr$   c           	     `   d fd| D             }fd|D             }||k    s5t          d t          t          | |          |          D                       rt          d          d t	          |          D             }d t          | |          D             }d t          ||          D             }d	 |D             }d
 |D             }||k    rt          d|d|          t          |d          }	t          |d          }
dgt          |           z  }t          ||	|
          D ]&\  }}}t          t          ||                    ||<   't	          |          D ]-\  }}|r&| |         }d t	          |          D             }|||<   .t          d |D                       sJ |S )a  Helper to build old_chunks to new_chunks.

    Handles missing values, as long as the dimension with the missing chunk values
    is unchanged.

    This function expects that the arguments have been pre-processed by
    :func:`dask.array.core.normalize_chunks`. In particular any ``nan`` values should
    have been replaced (and are so by :func:`dask.array.core.normalize_chunks`)
    by the canonical ``np.nan``.

    Examples
    --------
    >>> old = ((10, 10, 10, 10, 10), )
    >>> new = ((25, 5, 20), )
    >>> old_to_new(old, new)  # doctest: +NORMALIZE_WHITESPACE
    [[[(0, slice(0, 10, None)), (1, slice(0, 10, None)), (2, slice(0, 5, None))],
      [(2, slice(5, 10, None))],
      [(3, slice(0, 10, None)), (4, slice(0, 10, None))]]]
    c                4    t          d | D                       S )Nc              3  >   K   | ]}t          j        |          V  d S Nmathisnanr   chunks     r"   	<genexpr>z1old_to_new.<locals>.is_unknown.<locals>.<genexpr>   s,      664:e$$666666r$   any)dims    r"   
is_unknownzold_to_new.<locals>.is_unknown   s    66#666666r$   c                &    g | ]} |          S r%   r%   r   rW   rX   s     r"   r#   zold_to_new.<locals>.<listcomp>   !    <<<#jjoo<<<r$   c                &    g | ]} |          S r%   r%   rZ   s     r"   r#   zold_to_new.<locals>.<listcomp>   r[   r$   c              3  (   K   | ]\  }}||k    V  d S rN   r%   )r   newolds      r"   rT   zold_to_new.<locals>.<genexpr>   s;       / /sCs
/ / / / / /r$   zoChunks must be unchanging along dimensions with missing values.

A possible solution:
  x.compute_chunk_sizes()c                    g | ]	\  }}||
S r%   r%   )r   iunknowns      r"   r#   zold_to_new.<locals>.<listcomp>   s!    VVVzq'gVVVVr$   c                    g | ]	\  }}||
S r%   r%   r   rW   rb   s      r"   r#   zold_to_new.<locals>.<listcomp>   "    XXXgPWXXXXr$   c                    g | ]	\  }}||
S r%   r%   rd   s      r"   r#   zold_to_new.<locals>.<listcomp>   re   r$   c                ,    g | ]}t          |          S r%   sum)r   r1   s     r"   r#   zold_to_new.<locals>.<listcomp>       +++AQ+++r$   c                ,    g | ]}t          |          S r%   rh   )r   r5   s     r"   r#   zold_to_new.<locals>.<listcomp>   rj   r$   zCannot change dimensions from z to r1   r5   Nc           	     f    g | ].\  }}|t          d t          j        |          s|nd          fg/S r   N)r8   rP   rQ   )r   jsizes      r"   r#   zold_to_new.<locals>.<listcomp>   sQ       At U1$*T*:*:DddEEFG  r$   c              3     K   | ]}|d uV  	d S rN   r%   r   xs     r"   rT   zold_to_new.<locals>.<genexpr>   s&      --q}------r$   )
rV   r   r   
ValueError	enumerater'   r   rJ   r.   all)
old_chunks
new_chunksold_is_unknownnew_is_unknownold_known_indices	old_known	new_known	old_sizes	new_sizescmoscmnsslicedra   cmocmnrb   rW   extrarX   s                     @r"   
old_to_newr      sM   *7 7 7 =<<<<<<N<<<<<<<N''3 / /#+C
J,G,G#X#X/ / / , ,' >
 
 	

 WVY~-F-FVVVXXZ)H)HXXXIXXZ)H)HXXXI+++++I+++++IIKYKKiKK
 
 	
 C((DC((DVc*oo%F,dD99 : :3!,sC"8"899q		//  
7 	Q-C (~~  E F1I--f--------Mr$   c                f    t          t          | |           }t          d |D                       }|S )a  
    Make dask.array slices as intersection of old and new chunks.

    >>> intersections = intersect_chunks(((4, 4), (2,)),
    ...                                  ((8,), (1, 1)))
    >>> list(intersections)  # doctest: +NORMALIZE_WHITESPACE
    [(((0, slice(0, 4, None)), (0, slice(0, 1, None))),
      ((1, slice(0, 4, None)), (0, slice(0, 1, None)))),
     (((0, slice(0, 4, None)), (0, slice(1, 2, None))),
      ((1, slice(0, 4, None)), (0, slice(1, 2, None))))]

    Parameters
    ----------

    old_chunks : iterable of tuples
        block sizes along each dimension (convert from old_chunks)
    new_chunks: iterable of tuples
        block sizes along each dimension (converts to new_chunks)
    c              3  B   K   | ]}t          t          |           V  d S rN   )r   r   )r   crs     r"   rT   z#intersect_chunks.<locals>.<genexpr>   s.      77"%%%777777r$   )r   r   r   )rv   rw   cross1crosss       r"   intersect_chunksr      s9    ( jZ889F7777777ELr$   autoFc                     j         dk    r t          d  j        D                       r S t          |t                    rd fd|                                D             }t           j                   D ]/}||vr j        |         ||<   ||          j        |         ||<   0t          |t          t          f          r,t          d t          | j                  D                       }t          | j        | j         j                  } j         }t          |          |k    st          d          |s| j        k    r S |rt          d |D                       }t          t          t           |                    }t          | j                  D ]B\  }	}
|	|
k    r7t#          j        |
          s#t#          j        |	          st          d          C|pt'          j        d	          }|d
k    r9t+           j        | j        j        ||          }|D ]}t/           |            S |dk    rddlm}  | |          S t5          d| d          )a  
    Convert blocks in dask array x for new chunks.

    Parameters
    ----------
    x: dask array
        Array to be rechunked.
    chunks:  int, tuple, dict or str, optional
        The new block dimensions to create. -1 indicates the full size of the
        corresponding dimension. Default is "auto" which automatically
        determines chunk sizes.
    threshold: int, optional
        The graph growth factor under which we don't bother introducing an
        intermediate step.
    block_size_limit: int, optional
        The maximum block size (in bytes) we want to produce
        Defaults to the configuration value ``array.chunk-size``
    balance : bool, default False
        If True, try to make each chunk to be the same size.

        This means ``balance=True`` will remove any small leftover chunks, so
        using ``x.rechunk(chunks=len(x) // N, balance=True)``
        will almost certainly result in ``N`` chunks.
    method: {'tasks', 'p2p'}, optional.
        Rechunking method to use.


    Examples
    --------
    >>> import dask.array as da
    >>> x = da.ones((1000, 1000), chunks=(100, 100))

    Specify uniform chunk sizes with a tuple

    >>> y = x.rechunk((1000, 10))

    Or chunk only specific dimensions with a dictionary

    >>> y = x.rechunk({0: 1000})

    Use the value ``-1`` to specify that you want a single chunk along a
    dimension or the value ``"auto"`` to specify that dask can freely rechunk a
    dimension to attain blocks of a uniform block size

    >>> y = x.rechunk({0: -1, 1: 'auto'}, block_size_limit=1e8)

    If a chunk size does not divide the dimension then rechunk will leave any
    unevenness to the last chunk.

    >>> x.rechunk(chunks=(400, -1)).chunks
    ((400, 400, 200), (1000,))

    However if you want more balanced chunks, and don't mind Dask choosing a
    different chunksize for you then you can use the ``balance=True`` option.

    >>> x.rechunk(chunks=(400, -1), balance=True).chunks
    ((500, 500), (1000,))
    r   c              3  "   K   | ]
}|d k    V  dS rm   r%   )r   ss     r"   rT   zrechunk.<locals>.<genexpr><  s&      22Q!q&222222r$   c                B    i | ]\  }}t          |j                  |S r%   )r   ndim)r   cvrr   s      r"   
<dictcomp>zrechunk.<locals>.<dictcomp>@  s+    III$!Q-16**AIIIr$   Nc              3  (   K   | ]\  }}||n|V  d S rN   r%   )r   lcrcs      r"   rT   zrechunk.<locals>.<genexpr>G  s.      XXBR^rrXXXXXXr$   )limitdtypeprevious_chunksz-Provided chunks are not consistent with shapec              3  4   K   | ]}t          |          V  d S rN   )_balance_chunksizesrR   s     r"   rT   zrechunk.<locals>.<genexpr>U  s+      FFe*511FFFFFFr$   zarray.rechunk.methodtasksp2p)rechunk_p2pzUnknown rechunking method '')r   ru   shape
isinstancedictitemsr6   r&   r   listr   r   r   r   rs   mapri   rP   rQ   r   getplan_rechunkitemsize_compute_rechunkdistributed.shuffler   NotImplementedError)rr   r&   	thresholdblock_size_limitbalancemethodra   r   
new_shapesr^   r_   stepsr   r   s   `             r"   rechunkr      s   F 	vzzc22!'22222z&$ (IIII&,,..IIIqv 	( 	(AHQKq		"HQKq	&5$-(( YXX#fahBWBWXXXXX/qwPQPX  F
 6Dv;;$HIII !(** GFFvFFFFFs3''((J
AG,, N NS#::djoo:djoo:LMMM9vz"899FHfag.	;K
 
  	' 	'A A&&AA	5333333{1f%%% ""I"I"I"IJJJr$   c                R    t          t          t          t          |                     S rN   )r   r   r   r   r&   s    r"   _number_of_blocksr   q      #s3''(((r$   c                R    t          t          t          t          |                     S rN   )r   r   r   maxr   s    r"   _largest_block_sizer   u  r   r$   c                `    t          t          d t          | |          D                       }|S )z5Estimate the graph size during a rechunk computation.c              3     K   | ]>\  }}||k    r"t          |          t          |          z   d z
  nt          |          V  ?dS r   Nr   )r   ocncs      r"   rT   z&estimate_graph_size.<locals>.<genexpr>  sc       	
 	
B ')BhhSWWs2ww""CGG	
 	
 	
 	
 	
 	
r$   )r   r   r   )rv   rw   crossed_sizes      r"   estimate_graph_sizer   y  sD     	
 	
j*55	
 	
 	
 L r$   c                    g }| D ]b}t          t          j        ||z                      }t          |          D ]$}|||z
  z  }|                    |           ||z  }%|dk    sJ ct          |          S )zpMinimally divide the given chunks so as to make the largest chunk
    width less or equal than *max_width*.
    r   )intnpceilr6   r7   r   )desired_chunks	max_widthr&   r   
nb_dividesra   r5   s          r"   divide_to_widthr     s     F  Y//00
z"" 	 	Aj1n%AMM!FAAAvvvvv==r$   c                    t                     |k    r S t                     }t          |          dk    rS|                                }t                     }||z  }||z  }|||z  z  }|||z  z
  |z  }||z   f|z  |f||z
  z  z   S t                     |z  }t                     |z
  }	 fdt	          t                     dz
            D             }
t          j        |
           t                     }|	dk    rt          j        |
          \  }}}||         dk    rJ|dz  }||         dk    r|dz  }||         dk    t          j	        |
||         ||         z   ||f           t||         ||         z   |k    r(t          j	        |
||         ||         z   ||f           ||         dk    sJ d||<   |||<   |	dz  }	|	dk    t          t          d|                    S )zMinimally merge the given chunks so as to drop the number of
    chunks below *max_number*, while minimizing the largest width.
    r   c                D    g | ]}|         |d z            z   ||d z   fS r   r%   )r   ra   r   s     r"   r#   z#merge_to_number.<locals>.<listcomp>  sG        
	^AE2	2Aq1u=  r$   r   N)r   setpopri   r6   heapqheapifyr   heappopheappushr   filter)r   
max_numberdistinctwr5   totaldesired_widthwidthadjustnmergesheapr&   ra   rn   s   `             r"   merge_to_numberr     sD    >j((>""H
8}}LLNNA+]a'(*u,,2	|f$x:3F'GGG'':5M.!!J.G   s>**Q.//  D 
M$.!!F
A++mD))q! !9>>FA)q..Q )q..N4&)fQi"7A!>???AY"e++N4&)fQi"7A!>???ayA~~~~q	q	1% A++( f%%&&&r$   c                2   t          |           }d | D             d |D             d t          t          | |                    D             fdt          |          D             fdt          |          D             }fd}t	          ||          }t          t                    }t          |           }d}	|D ]}
||
         z  |
         pd	z  }||k    r||
         ||
<   |},|
         }t          ||z  |z            }t          ||
         |          }t          |          t          | |
                   k    r|||
<   |t          |          z  |z  }d
}	|t          |          k    sJ ||k    sJ t          |          |	fS )z
    Find an intermediate rechunk that would merge some adjacent blocks
    together in order to get us nearer the *new_chunks* target, without
    violating the *block_size_limit* (in number of elements).
    c                ,    g | ]}t          |          S r%   r   r   r   s     r"   r#   z&find_merge_rechunk.<locals>.<listcomp>      444AQ444r$   c                ,    g | ]}t          |          S r%   r   r   s     r"   r#   z&find_merge_rechunk.<locals>.<listcomp>  r   r$   c                Z    i | ](\  }\  }}|t          |          t          |          z  )S r%   r   )r   rW   r   r   s       r"   r   z&find_merge_rechunk.<locals>.<dictcomp>  sC       C"b 	SWWs2ww  r$   c                8    i | ]}||         |         pd z  S r   r%   )r   rW   new_largest_widthold_largest_widths     r"   r   z&find_merge_rechunk.<locals>.<dictcomp>  sA        	s#'8'='BC  r$   c                ,    g | ]}|         d k    |S )g      ?r%   )r   rW   graph_size_effects     r"   r#   z&find_merge_rechunk.<locals>.<listcomp>  s*    TTT6G6LPS6S6S6S6S6Sr$   c                    |          }|          }|dk    rd}|dk    r)t          j        |          t          j        |          z  ndS )Nr   g0D   ?r   )r   log)kgsebseblock_size_effectr   s      r"   r*   zfind_merge_rechunk.<locals>.key  sM    ""!88C.1AggsbfSkk))1<r$   r)   Fr   T)r   rt   r   r6   r+   r   r   r   r   r   r   r   r   )rv   rw   r   r   merge_candidatesr*   sorted_candidateslargest_block_sizer&   memory_limit_hitrW   new_largest_block_sizelargest_widthchunk_limitr   r   r   r   r   s                  @@@@r"   find_merge_rechunkr     s/    z??D4444444444 &s:z'B'BCC  
    ;;   UTTTuT{{TTT= = = = = = /S999%677*F  $ $ !23!77<Mc<R<WVWX 	 "%555$S/F3K!7 .c2M.>ASSTTK
3==A1vvZ_----s%7#a&&%@M%Q"#!4V!<!<<<<<!11111==***r$   c                R   t          |           }t          |           }t          |          D ]}t          ||          }||k    r nt          | |                   t          ||                   k    rGt	          t          | |                   |z  |z            }t          ||         |          }t          |          |k    sJ t          |          t          | |                   k    r+t          |          t          | |                   k    r|||<   t          |          S )z
    Find an intermediate rechunk that would split some chunks to
    get us nearer *new_chunks*, without violating the *graph_size_limit*.
    )r   r   r6   r   r   r   r   r   )	rv   rw   graph_size_limitr   r&   rW   
graph_sizer   r   s	            r"   find_split_rechunkr     s   
 z??D*FT{{  (<<
(((Ez##jo"6"666Z_--0@@:MNN
JsOZ881vv#### q66SC))))c!ffJsO8L8L.L.LF3K==r$   c                   |pt          j        d          }|pt          j        d          }t          |t                    rt	          |          }d | D             }t          |          dk    st          |          rt          |          r|gS ||z  }t          |           }t          |          }t          |||g          }|t          |           t          |          z   z  }| }	d}
g }	 t          |	|          }||k     rn_|
r|	}nt          |	|||z            }t          |||          \  }}||	k    r|
r||k    rn#||	k    r|                    |           |}	|snd}
v||gz   S )a3  Plan an iterative rechunking from *old_chunks* to *new_chunks*.
    The plan aims to minimize the rechunk graph size.

    Parameters
    ----------
    itemsize: int
        The item size of the array
    threshold: int
        The graph growth factor under which we don't bother
        introducing an intermediate step
    block_size_limit: int
        The maximum block size (in bytes) we want to produce during an
        intermediate step

    Notes
    -----
    No intermediate steps will be planned if any dimension of ``old_chunks``
    is unknown.
    zarray.rechunk-thresholdzarray.chunk-sizec              3  H   K   | ]}t          d  |D                       V  dS )c              3  >   K   | ]}t          j        |          V  d S rN   rO   )r   ys     r"   rT   z)plan_rechunk.<locals>.<genexpr>.<genexpr>L  s*      --aDJqMM------r$   NrU   rq   s     r"   rT   zplan_rechunk.<locals>.<genexpr>L  s9      BB!--1-----BBBBBBr$   r   TF)r   r   r   strr   r   ru   rV   r   r   r   r   r   r   r7   )rv   rw   r   r   r   has_nanslargest_old_blocklargest_new_blockgraph_size_thresholdcurrent_chunks
first_passr   r   r&   r   s                  r"   r   r   1  s   , BVZ(ABBI'I6:6H+I+I"C(( 9&'788BBzBBBH
:!3z??c(mm|   ,J77+J77,.?ARSTT %*%%(9*(E(EE  NJE(DD
,,, 
	#FF (
J,B F $6J 0$
 $
   n$$Z$Fj<P<P^##LL    	
58 J<r$   c                4     j         dk    rt           j        | j                  S  j        }t           j        |          }t                      }t                      }t           |          }d|z   }d|z   }t                      }	t          j        d  j        D             d          }
t          j        |
j                  D ]} j        f|z   |
|<   t          d |D              }t          ||          D ]Y\  }|f|z   }fd	t          |          D             fd
t          |          D             }t          j        |d          }|j        }t#                    D ]\  }}t          | \  }}|t%          |	          f}|
|         dd         }t'           fdt#          t          ||                    D                       r|
|         ||<   tt(          |
|         |f||<   |||<   ||j         dz
  k    sJ t'          d |j        D                       r|j        d         ||<   ;t*          |                                f||<   [~
~t/          j        ||          }t3          j        || g          }t7          |||           S )z1Compute the rechunk of *x* to the given *chunks*.r   )r&   r   zrechunk-merge-zrechunk-split-c                ,    g | ]}t          |          S r%   r   r   s     r"   r#   z$_compute_rechunk.<locals>.<listcomp>  s    444a3q66444r$   O)r   c              3  N   K   | ] }t          t          |                    V  !d S rN   )r6   r   r   s     r"   rT   z#_compute_rechunk.<locals>.<genexpr>  s.      88A%A--888888r$   c                .    g | ]fd D             S )c                ,    g | ]}|         d          S r   r%   )r   r   ra   s     r"   r#   z/_compute_rechunk.<locals>.<listcomp>.<listcomp>  s!    8882beAh888r$   r%   )r   ra   r   s    @r"   r#   z$_compute_rechunk.<locals>.<listcomp>  s/    NNNQ8888888NNNr$   c                T    g | ]$}t          t          |                             %S r%   )r   r   )r   ra   old_block_indicess     r"   r#   z$_compute_rechunk.<locals>.<listcomp>  s/    HHHqC-a01122HHHr$   r   Nc              3  r   K   | ]1\  }\  }}|j         d k    o|j        j        |         |         k    V  2dS rm   )r=   stopr&   )r   ra   rI   indrr   s       r"   rT   z#_compute_rechunk.<locals>.<genexpr>  s\        !AzS 	Q?38qx{3/?#?     r$   c              3  "   K   | ]
}|d k    V  dS r   r%   )r   ds     r"   rT   z#_compute_rechunk.<locals>.<genexpr>  s&      11!qAv111111r$   )dependencies)meta)ro   r   r   r   r   r   r&   r   r   r   r   ndindexnamer   r   r6   flatrt   nextru   r   r   tolisttoolzmerger   from_collectionsr   )rr   r&   r   crossedx2intermediatestoken
merge_name
split_namesplit_name_suffixes
old_blocksindex	new_indexnew_idxr*   subdims1rec_cat_argrec_cat_arg_flatrec_cat_index
ind_slicesold_block_indexslicesr  	old_indexlayergraphr   r  s   `                         @@r"   r   r     s   v{{QWV17;;;;6Dqx00G	BFFMQE!E)J!E)J'' 4418444C@@@JJ,-- . .VI-
5 888889Iy'22 ; ;mg%NNNN%++NNNHHHHE$KKHHHhxs333&+ *36):): 	7 	7%M:&):&6#OV%8 9 9:D"?3ABB7I    %.s69/E/E%F%F     7 3=_2M //'.
?0KV&Td#26 // 01 44444 11{011111 	;!&q)BsGG#[%7%7%9%9:BsGGIKM**E+JQCPPPE
F3333r$   c                      e Zd Zd Zd ZeZdS )_PrettyBlocksc                    || _         d S rN   blocks)selfr8  s     r"   __init__z_PrettyBlocks.__init__  s    r$   c                   g }g }d}| j         D ]}|rT|d         |k    rH|dk    r<t          |          dk    r)|                    d |d d         f           |dd          }|dz  }X|dk    r9t          |          dk    sJ |                    |dz   |d         f           g }d}|                    |           |rS|dk    r|                    d |f           n5t          |          dk    sJ |                    |dz   |d         f           g }|D ]D\  }}|#|                    t          |                     *|                    d||fz             Ed                    |          S )Nr   r4   r   z%d*[%s]z | )r8  r   r7   r  join)r9  runsrunrepeatsr   partss         r"   __str__z_PrettyBlocks.__str__  s    	 	A s2w!||a<<CHHqLLKKs3B3x 0111bcc(C1Q;;s88q====KK1c"g 6777CG

1 	4!||T3K((((3xx1}}}}Wq[#b'2333  	9 	9LGSSXX&&&&Y'378888zz%   r$   N)__name__
__module____qualname__r:  rA  __repr__r%   r$   r"   r5  r5    s4          ! ! !@ HHHr$   r5  c                    t          | t                    rt          d | D                       sJ t          |           S )z
    Pretty-format *blocks*.

    >>> format_blocks((10, 10, 10))
    3*[10]
    >>> format_blocks((2, 3, 4))
    [2, 3, 4]
    >>> format_blocks((10, 10, 5, 6, 2, 2, 2, 7))
    2*[10] | [5, 6] | 3*[2] | [7]
    c              3  h   K   | ]-}t          |t                    pt          j        |          V  .d S rN   )r   r   rP   rQ   rq   s     r"   rT   z format_blocks.<locals>.<genexpr>  sJ       - -01
1c+djmm- - - - - -r$   )r   r   ru   r5  r7  s    r"   format_blocksrH    s]     fe$$  - -5;- - - * *       r$   c                b    t          | t                    sJ t          d | D                       S )zH
    >>> format_chunks((10 * (3,), 3 * (10,)))
    (10*[3], 3*[10])
    c              3  4   K   | ]}t          |          V  d S rN   )rH  r   s     r"   rT   z format_chunks.<locals>.<genexpr>  s*      22aq!!222222r$   )r   r   r   s    r"   format_chunksrK    s7    
 fe$$$$$226222222r$   c                    d | D             S )zs
    >>> format_plan([((10, 10, 10), (15, 15)), ((30,), (10, 10, 10))])
    [(3*[10], 2*[15]), ([30], 3*[10])]
    c                ,    g | ]}t          |          S r%   )rK  r   s     r"   r#   zformat_plan.<locals>.<listcomp>  s     +++M!+++r$   r%   )plans    r"   format_planrO     s    
 ,+d++++r$   c                n    | |z  }| |z  }|g|z  }|r|                     |           t          |          S rN   )r7   r   )r5   	chunksizeleftovern_chunksr&   s        r"   _get_chunksrT    sF    9}HI~H[8#F  h==r$   r&   tuple[int, ...]returnc                    t          j                                       t                    }t	                     |dz  }t                     dt                     z  k    rdz   fdt          ||z
  ||z   dz             D             }fd|D             }t	          |          st          d            S d |D             }t          j	        |          }||         S )z
    Balance the chunk sizes

    Parameters
    ----------
    chunks : tuple[int, ...]
        Chunk sizes for Dask array.

    Returns
    -------
    new_chunks : tuple[int, ...]
        New chunks for Dask array with balanced sizes.
    r3   g      ?r   c                J    g | ]}t          t                    |           S r%   )rT  ri   )r   	chunk_lenr&   s     r"   r#   z'_balance_chunksizes.<locals>.<listcomp>&  s9        	CKK++  r$   c                :    g | ]}t          |          k    |S r%   r   )r   r   rS  s     r"   r#   z'_balance_chunksizes.<locals>.<listcomp>*  s)    CCCQA(0B0Bq0B0B0Br$   zSchunk size balancing not possible with given chunks. Try increasing the chunk size.c                L    g | ]!}t          |          t          |          z
  "S r%   )r   minr   s     r"   r#   z'_balance_chunksizes.<locals>.<listcomp>2  s(    666SVVc!ff_666r$   )
r   medianastyper   r   r\  r   r6   r   argmin)r&   
median_lenepsrw   possible_chunksdiffsbest_chunk_sizerS  s   `      @r"   r   r     s    6""))#..J6{{H
/C
6{{cCKK'''A   zC/c1AA1EFF  J DCCC*CCCO -	
 	
 	
 66o666Ei&&O?++r$   )r   NNFN)NN)r&   rU  rV  rU  )>__doc__
__future__r   r   rP   	functoolsr   	itertoolsr   r   r   r   operatorr	   r
   r   warningsr   numpyr   tlzr  r   daskr   dask.array.chunkr   dask.array.corer   r   r   dask.array.utilsr   dask.array.wrapr   	dask.baser   dask.highlevelgraphr   
dask.utilsr   r'   r.   rJ   r   r   r   r   r   r   r   r   r   r   r   r   r5  rH  rK  rO  rT  r   r%   r$   r"   <module>ru     s    # " " " " "         5 5 5 5 5 5 5 5 5 5 5 5 ) ) ) ) ) ) ) ) ) )                           $ $ $ $ $ $ A A A A A A A A A A * * * * * * ! ! ! ! ! !       . . . . . . " " " " " "
 
 
= = =` ` `F@ @ @F  6 uK uK uK uKp) ) )) ) )    3' 3' 3'lF+ F+ F+R  : HLN  N  N  N b:4 :4 :4z$ $ $ $ $ $ $ $N! ! !"3 3 3, , ,  ", ", ", ", ", ",r$   