
    Kgz                   Z   d Z ddlmZ ddlZddlZddlZddlZddlZddlZddl	m
Z ddl	mZ ddlmZmZ ddlmZ ddlZddlmZ dd	lmZ dd
lmZmZ ddlmZ ddlmZ ddl m!Z! ddl"m#Z# ddl$m%Z%m&Z&m'Z' ddl(m)Z) ddl*m+Z+ ddl,m-Z-m.Z.m/Z/ ddl0m1Z1m2Z2m3Z3m4Z4m5Z5 ddl6m7Z7 ddl8m9Z9m:Z:m;Z;m<Z<m=Z=m>Z> ddl?m@Z@ dZAdZBdZCeCrddl$mDZD erddlEmFZF ddl"mGZG ddlHmIZImJZJ dZKdZL edddd       ZMd!ZNd"ZOd,d#ZP G d$ d%ee#ej                        ZQ G d& d'ee#      ZR G d( d)ee#      ZS G d* d+ee#      ZTy)-z Here is defined the Index class.    )annotationsN)perf_counter)process_time)LiteralTYPE_CHECKING)Path   )indexesextension)UIntAtomAtom)Filters)NotLoggedMixin)	join_path)Group)is_idxidx2longlazyattr)CArray)EArray)
CacheArrayLastRowArray
IndexArray)calc_chunksizecalcoptlevelsget_reduction_level	nextafterinftype)PerformanceWarning)nan_aware_gtnan_aware_genan_aware_ltnan_aware_lebisect_leftbisect_right)ObjectCachez2.1F)
show_stats)Array)	RootGroup)ColsTable	mergesortTzlib)	complevelcomplibshuffle
fletcher32)
int8int16int32int64uint8uint16uint32uint64float32float64l        c                    | j                  d      }t        |      D ]  \  }}|j                  d      s n dj                  |d        dz   dd  z   }dj                  ||dz   d        }||fS )N/_i_   r	   )split	enumerate
startswithjoin)indexpathnamenamesinametablepathnamecolpathnames         ,lib/python3.12/site-packages/tables/index.py_table_column_pathname_of_indexrJ   g   s}    $EU#4??5! $ HHU2AY'#-QR8M((5Q>*K;''    c                      e Zd ZdZdZed?d       Zed@d       ZedAd       Zej                  dBd       ZedCd       Z
edDd       ZedEd	       ZedEd
       ZedEd       ZedEd       ZedEd       ZedEd       ZedEd       ZedFd       ZedAd       ZedAd       ZedAd       ZedEd       ZedEd       Z	 	 	 	 	 	 	 	 	 	 dG	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dH fdZdI fdZ	 	 	 	 	 	 	 	 dJdZdKdZdLdMdZ	 dL	 	 	 	 	 dMdZdLdNdZ dIdZ!	 dO	 	 	 	 	 dPdZ"dIdZ#dId Z$dId!Z%	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dQd"Z&	 dR	 	 	 dSd#Z'	 dT	 	 	 	 	 	 	 	 	 dUd$Z(	 dT	 	 	 	 	 	 	 	 	 dUd%Z)	 dT	 	 	 	 	 	 	 dVd&Z*	 dT	 	 	 	 	 	 	 dVd'Z+	 dT	 	 	 	 	 	 	 dVd(Z,	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dWd)Z-dXd*Z.dId+Z/	 dR	 	 	 dSd,Z0	 dT	 	 	 	 	 	 	 	 	 	 	 dYd-Z1	 	 	 	 	 	 	 	 dZd.Z2	 	 	 	 	 	 	 	 dZd/Z3	 	 	 	 	 	 	 	 	 	 d[d0Z4	 	 	 d\	 	 	 	 	 	 	 d]d1Z5	 	 	 d\	 	 	 	 	 	 	 d]d2Z6	 	 	 	 	 	 	 	 d^d3Z7d_d4Z8dEd5Z9dId6Z:d`d7Z;	 	 	 	 	 	 dad8Z<dbd9Z=dcd:Z>	 	 	 	 	 	 ddd;Z?dLde fd<Z@dfd=ZAdfd>ZB xZCS )gIndexa  Represents the index of a column in a table.

    This class is used to keep the indexing information for columns in a Table
    dataset (see :ref:`TableClassDescr`). It is actually a descendant of the
    Group class (see :ref:`GroupClassDescr`), with some added functionality. An
    Index is always associated with one and only one column in the table.

    .. note::

        This class is mainly intended for internal use, but some of its
        documented attributes and methods may be interesting for the
        programmer.

    Parameters
    ----------
    parentnode
        The parent :class:`Group` object.

        .. versionchanged:: 3.0
           Renamed from *parentNode* to *parentnode*.

    name : str
        The name of this node in its parent group.
    atom : Atom
        An Atom object representing the shape and type of the atomic objects to
        be saved. Only scalar atoms are supported.
    title
        Sets a TITLE attribute of the Index entity.
    kind
        The desired kind for this index.  The 'full' kind specifies a complete
        track of the row position (64-bit), while the 'medium', 'light' or
        'ultralight' kinds only specify in which chunk the row is (using
        32-bit, 16-bit and 8-bit respectively).
    optlevel
        The desired optimization level for this index.
    filters : Filters
        An instance of the Filters class that provides information about the
        desired I/O filters to be applied during the life of this object.
    tmp_dir
        The directory for the temporary files.
    expectedrows
        Represents an user estimate about the number of row slices that will be
        added to the growable dimension in the IndexArray object.
    byteorder
        The byteorder of the index datasets *on-disk*.
    blocksizes
        The four main sizes of the compound blocks in index datasets (a low
        level parameter).
    new
        Whether this Index is new or has to be read from disk.

    INDEXc                *    ddddd| j                      S )zIndex kind.
ultralightlightmediumfullr	            )indsizeselfs    rI   kindz
Index.kind   s!      GVDLL
 	
rK   c                    | j                   S )z\Filter properties for this index.

        See Filters in :ref:`FiltersClassDescr`.
        )
_v_filtersrY   s    rI   filterszIndex.filters   s     rK   c                0    t        | j                  dd      S )zWhether the index is dirty or not.

        Dirty indexes are out of sync with column data, so they exist but they
        are not usable.
        DIRTYF)getattr_v_attrsrY   s    rI   dirtyzIndex.dirty   s     t}}gu55rK   c                    | j                   t        |      }}|| j                  _        | j                  j
                  }|s|r|j                          |r|s|j                          y y y N)rc   boolrb   r`   table_condition_cachenailunnail)rZ   rc   wasdirtyisdirtyconditioncaches        rI   rc   zIndex.dirty   sY     JJU'# 44G!G!!# $8rK   c                    t        | j                        \  }}| j                  j                  |      }|j                  j                  |      }|S )zVColumn instance for the indexed column.

        See :ref:`ColumnClassDescr`.
        )rJ   _v_pathname_v_file	_get_nodecols_g_col)rZ   	tablepath
columnpathrg   columns        rI   rv   zIndex.column   sL     !@!
	: &&y1"":.rK   c                l    t        | j                        \  }}| j                  j                  |      }|S )z.Accessor for the `Table` object of this index.)rJ   ro   rp   rq   )rZ   rt   ru   rg   s       rI   rg   zIndex.table   s8     !@!
	: &&y1rK   c                4    | j                   | j                  z  S )z8Property providing the number of blocks in a superblock.)superblocksize	blocksizerY   s    rI   nblockssuperblockzIndex.nblockssuperblock   s     ""dnn44rK   c                4    | j                   | j                  z  S )z3Property providing the number of slices in a block.)rz   	slicesizerY   s    rI   nslicesblockzIndex.nslicesblock        ~~//rK   c                4    | j                   | j                  z  S )z3Property providing the number of chunks in a slice.)r}   	chunksizerY   s    rI   nchunkslicezIndex.nchunkslice   r   rK   c                    | j                   | j                  z
  }|| j                  z  }|| j                  z  dkD  r|dz  }|S )z%Total number of superblocks in index.r   r	   )	nelementsnelementsILRry   rz   rZ   r   nblockss      rI   nsuperblockszIndex.nsuperblocks   sG     NNT%6%66	t222t~~%)qLGrK   c                    | j                   | j                  z
  }|| j                  z  }|| j                  z  dkD  r|dz  }|S )z Total number of blocks in index.r   r	   )r   r   rz   r   s      rI   r   zIndex.nblocks   sE     NNT%6%66	t~~-t~~%)qLGrK   c                4    | j                   | j                  z  S )z.Return the number of complete slices in index.)r   r}   rY   s    rI   nsliceszIndex.nslices  r   rK   c                4    | j                   | j                  z  S )z.Return the number of complete chunks in index.)r   r   rY   s    rI   nchunkszIndex.nchunks  r   rK   c                2    | j                   | j                  fS )z-Shape of this index (in slices and elements).)nrowsr}   rY   s    rI   shapezIndex.shape  s     zz4>>))rK   c                    | j                   dkD  xr4 | j                  dkD  xr# | j                  j                  | j                  kD  S )z8Whether a temporary file for indexes is required or not.r	   r   )rX   optlevelrg   r   r}   rY   s    rI   temp_requiredzIndex.temp_required  sA     LL1 2!2

  4>>1	
rK   c                B    | j                   dk(  xr | j                  dk(  S )z@Whether we should try to build a completely sorted index or not.rW   	   )rX   r   rY   s    rI   want_complete_sortzIndex.want_complete_sort  s!     ||q 7T]]a%77rK   c                    | j                   dk(  ry| j                  dk  ryd| j                  v r| j                  j                  S | j	                  | dd       | j
                  dk(  S )zWhether the index is completely sorted or not.

        .. versionchanged:: 3.0
           The *is_CSI* property has been renamed into *is_csi*.

        r   FrW   is_csiN)r   rX   rb   r   compute_overlaps	noverlapsrY   s    rI   r   zIndex.is_csi$  sa     >>Q<<!t}}$==''' 	dD%0~~""rK   c                4    | j                   j                  d   S )z7Return the number of rows that fits in a *table* chunk.r   )rg   
chunkshaperY   s    rI   nrowsinchunkzIndex.nrowsinchunk:  s     zz$$Q''rK   c                d   t        | j                  | j                        }| j                  dk(  r8d}| j                  ||z  kD  r"t        j                  | j                  |z        }|S | j                  dk(  r8d}| j                  ||z  kD  r"t        j                  | j                  |z        }|S 	 |S )z/Return the length of a bucket based index type.r	      rU   i   )minr   r   rX   r}   mathceilrz   )rZ   lbucketmaxnbs      rI   r   zIndex.lbucket?  s     d''8<<1 E~~/))DNNU$:;  \\Q E~~/))DNNU$:;
  rK   c                >   d | _         	 || _        	 || _        	 t        |	      | _        	 |
dv r|
| _        nt        j
                  | _        	 |J|j                  j                  | _        |j                  | _	        	 ddddd|   }|dv sJ d       || _
        	 d | _        	 d | _        	 || _        	 d	| _        	 d | _        	 d | _        	 d | _        	 d | _        	 d | _        	 t(        | _        	 d
| _        	 d| _        	 ddlm} || _        	 t4        | m  |||||       y )N)littlebigr	   rU   rV   rW   )rP   rQ   rR   rS   rT   zindsize should be 1, 2, 4 or 8!Tr   )	open_file)
_v_versionr   tmp_dirintexpectedrows	byteordersysdtypebasetyperX   r   r   
blocksizes
dirtycachery   rz   r}   r   tmpfilenameopt_search_typesr   tproffiler   	_openFilesuper__init__)rZ   
parentnoderF   atomtitler[   r   r^   r   r   r   r   newrX   r   	__class__s                  rI   r   zIndex.__init__V  sL     '+/ 44-;))&DN ]]DN2DJ		DIG &'aKG l*M,MM*"DLA!%
6%)C$H=*.4%)/%)/%)/'+, 0L	H
5#"ET5#w?rK   c                   | j                   rt        | _        t        |           | j                   s| j                  }| j
                  }t        |j                        | _        t        |j                        | _        t        |j                        | _	        t        |j                        | _
        | j                  | j                  | j                  | j                  f| _        t        |j                        | _        | j                  }| j                  }|j                  j                   | _        |j                  j"                  | _        |j                  j$                  | _        | j                  |j(                  d   k(  sJ d       | j                  |j*                  d   k(  sJ d       | j&                  dv sJ d       |dkD  rmt        |j,                        | _        t        | j.                  j0                  j2                        }t        | j4                  j0                  j2                        }nd| _        | j4                  d   }|}|j6                  | _        | j6                  | j                  z  |z   | _        || _        || _        |dkD  r| xj6                  dz  c_        | j                  | j,                  z  }|dz
  |z  }|dk  rd}|d	z  }| j.                  |||z    | _        y d| _        d| _        d| _        d| _        t?        j@                  | j                         }	| jB                  }
| j                  2tE        | jF                  | j                  | j&                  | 
      | _        | j                  \  | _        | _        | _	        | _
        tH        rtK        d| j                         tM        | j&                  | j                  | j                  | j                        | _        | j                  | j,                  z  }| j                  | j,                  z  }tO        jP                  | j                        | j
                  _        tO        jP                  | j                        | j
                  _        tO        jR                  | j                        | j
                  _	        tO        jR                  | j                        | j
                  _
        | j                  | j
                  _        | j,                  | j
                  _        tU        | d|	d|
| jV                        }tU        | dtY        | j&                        d|
| jV                         t[        | d|	dd|
| jF                  | j                  z  | jV                         t]        | d|	dd|
| jV                  d       |dz
  |z  }t[        | d|	d|fd|
| j^                  d|f| jV                  	       t]        | d|	dd|
| jV                  d       t]        | d|	dd|
| jV                  d       t]        | d |	dd!|
| jV                  d       |d	z   |z   f}ta        | d"|	|d#|
|f| jV                        }| j                  f}ta        | d$tY        | j&                        |d%|
| j                  f| jV                        }d|j0                  _        d|j0                  _        d | _        tO        jb                  | j6                  tN        jd                  &      | _3        	 tO        jb                  | j6                  tN        jd                  &      | _4        	 | jj                  r| jm                          y y )'Nr	   zWrong slicesizezWrong chunksizerT   zWrong indices itemsizez2.0r   r   rU   )nodezblocksizes:sortedzSorted ValuesindicesitemsizezNumber of chunk in tablerangesr   rU   zRange Values)r   mrangesr   Median rangesF)r   _logboundszBoundary ValuesaboundszStart boundszboundsz
End boundsmboundsMedian boundssortedLRzLast Row sorted values + bounds	indicesLRzLast Row indicesr   r   )7_v_new	obversionr   r   _g_post_init_hookrb   r   ry   rz   r}   r   r   r   r   r   r   r   r   r   rX   r   _v_chunkshape	reductionr   attrsr   r   r   nelementsSLRr   beboundsr   
from_dtyper^   r   r   debugprintr   npr8   r7   r   r   r   r   r   r   r   emptyr3   startslengthsr   create_temp)rZ   
idxversionr   sorted_r   nelements_slrnelements_ilr
rchunksize
nbounds_lrr   r^   
rslicesizenbounds_inslicer   	sorted_lr
indices_lrr   s                   rI   r   zIndex._g_post_init_hook  s'   ;;'DO!# {{JMME #&e&:&:";D 1DN 1DN 1DN##	DO  /DMkkGllG ++DJ))DI"<<00DL>>W]]1%55H7HH5'"7"7"::! !:<<</I1II/E!!$U__!5 #DMM$7$7$A$A B #DNN$8$8$B$B C!" $r 2 - DJ!ZZ$..8=HDN -D -Dq 

a
4>>9J'!+
:JA~
!OJ MM
 :DM  
 tzz* ,, ??",!!4==$,,TDO OO	
NNN-1,LL$--
 ^^t~~5
^^t~~5
 (*yy1D1D'E$"$))DNN";"$))DNN";"$))DNN";!%"&.. (D/7DNN

 	dll+&NN	
 	/nn		
 	nn		
 &>j8 LL nn
	
 	nn		
 	nn		
 	nn		
 a/13 -Mnn	
	 !!dll+^^nn	

 %&	!%&
"  hhTZZrxx@KxxdjjAJ  rK   c                &   t         r
t               }t         rt        d       |j                         }| j                  }| j
                  }| j                  }t         rt        d       |dk(  r1t        j                  dt        |      d      t        ||z        z   }	n|dk(  r"t        j                  dt        |      d      }	nt        j                  t        |      d	|dz  z        }	| j                  }
t        |
|||
z  z  |
z  z
        }d|	d| t        |||
      D ]  }||
z   d
z
  |
z  |	|||
z     |dk(  rS|| j                  z  |z  |
z  }|d|dz  z  k  sJ |	t        j                  |      j!                  |	j"                        z  }	|dk(  rS|dkD  rNt        |      |kD  sJ | j%                  | j&                  |d|        | j%                  | j(                  |	d|        t         rt        d       t+        j,                  ||	       |d   }|d
kD  rNt         rt        d       |dd|   j/                         }t         rt        d       |}t         rt        d       |dkD  rd| j0                  _        t         rt        d       |||	fS )z9Compute an initial indices arrays for data to be indexed.zEntering initial_appendzBefore creating idxrW   r   r8   r   rV   r7   zuint%dr	   rU   NzBefore keysortr   zBefore reductionzAfter reductionzAfter arr <-- reducFzExiting initial_append)profileclockr&   poprX   r}   r   r   arangelenr   r   r   ranger~   asarrayastyper   read_slice_lrr   r   r
   keysortcopyrb   r   )rZ   xarrnrowr   trefarrrX   r}   r   idxr   offsetrE   offset2larrreducs                   rI   initial_appendzIndex.initial_append  s|    7D0$7hhj,,NN	)),d3a<))As3xx83y < C \& ))As3xx8C((3s8X1%=>CllGTY-@%AW$LMNFC&M69g6()GaG'CAG$ 7!|  $"3"33y@GKw{!3333rzz'*11#))<<a<MA- s8m+++t}}c.=.ABt~~s>M/BC'.  c*2wq= -t4)$))+E,d3C0$7 !8#(DMM /6S#~rK   c                   t         r
t               }t         rt        d       |j                  d      }|t	        j
                  |      j                  |j                        z  }|t	        j
                  | j                        j                  |j                        z  }|j                  d      }t         rt        d       |S )z+Perform final operations in 32-bit indices.zEntering final_idx32r8   r7   zExiting final_idx32)r   r   r&   r   r   r   r   r   )rZ   r   r  r   s       rI   final_idx32zIndex.final_idx32  s    7D-t4jj"rzz&!((33 	

4<<(//		::jj",d3
rK   c                   t         r
t               }t         rt        d       |s| j                  r| j                  }d}n| }| j
                  }|j                  }|j                  }|j                  }|j                  }	|j                  }
|j                  }|j                  }|j                  }|j                  }|j                  }|j                   }| j#                  |||      \  }}}|j%                  |j'                  d|j(                               | j*                  |z  }| j,                  }|j%                  |d   |gg       |
j%                  ||d|   g       |j%                  |dd|          |j%                  ||dz
  d|          ||dz  d|   }|j%                  |       |	j%                  ||dz     g       t         rt        d       ~~t         rt        d       | j.                  dk(  r| j1                  ||| j2                  z        }|j%                  |j'                  d|j(                               t         rt        d	       ~|dz   | _        | j                   | j2                  z  | _        d| _        d| _        | j6                  |j:                  _        | j8                  |j:                  _        d
| _        t         rt        d       yy)z&Append the array to the index objects.zEntering appendr	   r   NrU   zBefore deleting arr & smedianzAfter deleting arr & smedianrV   zBefore deleting idxTzExiting append)r   r   r&   r   tmpr   r   r   r   r   r   r   r   r   r   r   r   r  appendreshapesizer   r   rX   r  r}   r   r   r   r   r   )rZ   r   updater   wherer   r   r   r   r   r   r   r   r   r   r   r   r  r   r   csncssmedians                          rI   r
  zIndex.append  so   7D($/$,,HHEIEI,,----------NN	__
,,T5)Dc3s{{1chh/0^^y(A~&'s26r6{m$s15b5z"s26<R<()bAgmm$wq)*+6=5t<<<1""3(>?Cs{{1chh/0,d3QY
dnn4 %)$5$5	!%)%6%6
"'. rK   c                N   t         r
t               }t         rt        d       | j                  }|s| j                  r| j
                  }d}n| }| j                  }|j                  }|j                  }| j                  |||      \  }	}
}t        |
      }t        |      }| j                  |z  }t        j                  |
dd|   |	gf      | _        ||j                  _        ||j                  _        t        | j                        }| j                  ||||z    |
|d| ~
| j"                  dk(  r| j%                  ||| j&                  z        }||dt        |       ~|dz   | _        || j&                  z  |z   | _        || _        || _        d| _        t         rt        d       yy)z/Append the array to the last row index objects.zEntering appendLRr	   NrV   TzExiting appendLR)r   r   r&   r   r   r	  r   r   r   r  r   r   r   concatenater   r   r   rX   r  r}   r   r   r   r   )rZ   r   r  r   r   r  r   r   r   r  r   r   r   r   r   r  s                   rI   append_last_rowzIndex.append_last_row3  s    7D*D1$,,HHEIEI__
NN	,,T5)Dc3CC^^y0
LjL(9D6'BC$1	!%2
" dmm$=A]]	--'"9:$'	.=!<<1""3(>?C!$
:SXQY
/-?)))40 rK   c                b   | j                   sy|rd| _        nt        | _        d| _        d| _        t        | j                  | j                  | j                        }|\  }}}}t        rt        d|       | j                          	 |rt        |      D ]u  }| j                  dd      r n| j                  dkD  r(| j                  dd      r n~| j                  dd      r nj| j                  dd	      r nV| j                  dd
      su nA n?|r| j                  dd      rn+|r| j                  dd	      rn|r| j                  dd
      rn	 | j                  rH| j                  dkD  r| j                          | j                  dkD  rt!        j"                  dt$               | j'                          y)zOptimize an index so as to allow faster searches.

        verbose
            If True, messages about the progress of the
            optimization process are printed out.

        NTr   z
optvalues:chunksmedianr	   slicesstartstopz[OPSI was not able to achieve a completely sorted index.  Please report this to the authors.)r   verboser   
last_tover
last_noverr   r   r   rX   r   create_temp2r   swapr   r   do_complete_sortwarningswarnUserWarningcleanup_temp)rZ   r  opts	optmedian	optstartsoptstopsoptfullniters           rI   optimizezIndex.optimized  s    !!DL DL  T\\4==$,,G26/	9h,%"7^Eyy84||a'  99Xx8!99Xx8!yy73yy62 , yy84yy73yy62 ""~~!%%'~~!; 	rK   c           	     \
   | j                   rt               }t               }| j                  }| j                  }|j
                  dd }| j                  }| j                  }|dkD  rNt        j                  | j                  d   | j                  d   g      }t        j                  ||gf      }|dz  }|j                  }	|j                  }
|j                  }|j                  }t        j                  g | j                         }t        j                  g d| j"                  z        }t        j$                  |t        j&                        }t)        |      D ]m  }t        j                  g | j                         }t        j                  g d| j"                  z        }||df   }t)        |dz   |      D ]  }||   }|| j                  k  r||k(  s|| j                  k(  r||k(  r2|| j                  k  r||k  sJ d       |	||f   }n||k  sJ d       ||   }||df   }||kD  r|| j                  k  rBt        j                  ||	||df   f      }t        j                  ||
||df   f      }|||<   |}t        j                  |||| f      }t        j                  |||| f      }|||<   ||kD  s| j+                  |||||   |      }|| j                  k  r=t        j                  ||	|||f   f      }t        j                  ||
|||f   f      }n4t        j                  |||| f      }t        j                  |||| f      }|||<    || j                  k  rEt        j                  ||	|||   df   |f      }t        j                  ||
|||   df   |f      }n<t        j                  ||||   | |f      }t        j                  ||||   | |f      }t-        j.                  ||       || j                  k  rN|d| |	|<   |d| |
|<   | j1                  ||d|        t3        |      t3        |      z   }||||z    }||||z    }t3        |      }||k(  sJ d}||d| ||d| |d| }t        j                  |dd| j4                     |d   gf      }||||t3        |      z    || _
        p dk(  sJ | j7                  | j                  d	| j                          | j                   r*t9        d
t               z
  ddt               z
  d       yy)z>Bring an already optimized index into a complete sorted state.Nr   r   r	   r   u%dr   z3Two slices cannot overlap completely at this stage!zdo_complete_sort()time: .4f	. clock: )r  r   cpuclockr}   r	  r   r   r   r   arrayr   r  r   r   r   r   r   rX   zerosint_r   search_item_ltr
   r   update_cachesr   r   r   r   )rZ   t1c1ssr	  r   r   nelements_lrrangeslrr   r   r   r   sremainiremainr   rE   soverioverprev_endjstjnext_begnext_endnr   ssortedsindicessendsortedlrr   s                                  rI   r   zIndex.do_complete_sort  sR   <<BB^^hhA,,((!xxq!14==3D EFH^^VhZ$89FqLG**++LL	]]
((2TZZ0((2UT\\%9:rww7wAHHRtzz2EHHRut||';<Ead|H1q5'*Qi$%#*= t||#bMLM &q#vH l*MLM*(~H!!Q$<h&4<<' "wq#$w7G/H I "wq#$w7G/H I$&q	( "yQ7G/H I "z#a7H/I J$0q	(--c8Qq	3OC4<<' "wq#c'z7J/K L "wq#c'z7J/K L "yS7I/J K "z#c7J/K L #F1IK +N 4<<..gan5u= >>gan5u= ..iq	LA5I >>j\BEJ $$Wh74<<$Sb\
%cr]
""1gcrl35zCL0!"rDy1"2T	2 LL((( '	"1!)
2A$]l3>>//08B<.A  ,H)EF !)k  p qyy
 	dhh(<dllK<<F57R<,Ihj2oc5JKL rK   c                   d}d}d}| j                   rt               }t               }|dk(  r| j                  |       n|dk(  r| j	                  |       |r
d| d| d}nd| }| j                  | j                  || j                         \  }	}
}t        |
j                         d	         t        |
      z  }| j                   r)t        d
t               z
  ddt               z
  d       |dk(  ra| j                  d	kD  rR| j                  d	kD  rC| j                  |z
  | j                  z  }| j                  |	z
  | j                  z  }|dk  r|dk  ry|| _
        |	| _        |	|k  ry||k  ryd	|cxk  r|k  ry yy)z7Swap chunks or slices using a certain bounds reference.(   g?g{Gz?r  r  swap_()r   r.  r/  r0  g?TF)r  r   r1  swap_chunksswap_slicesr   r	  r   nonzeror   r  r  )rZ   whatmodethnoverthmultthtoverr7  r8  messagenovermulttoverrmult	tover_var	nover_vars                  rI   r  z
Index.swap)  s    <<BB8T"XT"dV1TF!,GdVnG#44HHgt|| 
e DLLN1%&T2<<F57R<,Ihj2oc5JKL8! 3!8K50DOOCI50DOOCI4I$47?6>  rK   c           
        d| _         t        j                  dd| j                        \  }| _        t        j                  |       | j                  | j                  d      | _        | j                  j                  x| _
        }| j                  }| j                  }| j                  }d|f}t        j                  | j                         }t#        |d||d|d|f	       t#        |d
t%        | j&                        |d|d|f	       |dz
  |z  }d|f}t#        |d||d|||f	       d}t#        |d||d||f	       t#        |d||d||f	       t#        |d||d||f	       t#        |d|dd||df	       t#        |d|dd||f	       |dz   |z   f}t)        |d||d||f	       |f}t)        |dt%        | j&                        |d||f	       y )!9Create some temporary objects for slice sorting purposes.Tz.tmpz	pytables-wr   r   zTemporary sortedr	   r   r   r   zTemporary indicesr   zTemp chunk boundsr   r   zTemp start boundsr   zTemp end boundsr   r   r   r   zTemporary range valuesrU   r   r   r   z$Temp Last Row sorted values + boundsr   zTemp Last Row indicesN)rc   tempfilemkstempr   r   oscloser   tmpfilerootr	  r   r}   r^   r   r   r   r   r   rX   r   )	rZ   fdr	  r  r9  r^   r   r   r   s	            rI   r   zIndex.create_tempZ  s6    
'//K 
D 	~~d&6&6<***3^^^^,,Btzz*2w	
 	dll+2w	
 6b.O$O,	
 u	
 	u	
 	u	
 	$Aw	
 	u	
 a/)+2u	
 dll+#u	
rK   c           
        | j                   }| j                  }| j                  }| j                  |f}t	        j
                  | j                        }| j                  }t        |d||d|d|f       t        |dt        | j                        |d|d|f       |dz
  |z  }| j                  |f}t        |d||d	|||f       | j                  f}t        |d
||d||f       t        |d||d||f       t        |d||d||f       t        |d|| j                  dfd||df       t        |d|| j                  fd||f       y)r_  sorted2zTemporary sorted 2r	   ra  indices2r   zTemporary indices 2bounds2zTemp chunk bounds 2abounds2zTemp start bounds 2zbounds2zTemp end bounds 2mbounds2zMedian bounds 2ranges2rU   zTemporary range values 2mranges2zMedian ranges 2N)r   r}   r^   r   r   r   r   r	  r   r   rX   r   )rZ   r  r9  r^   r   r   r	  r   s           rI   r  zIndex.create_temp2  s    ^^^^,,r"tzz*hh 2w	
 	dll+!2w	
 6b./!O,	
 !u	
 	u	
 	u	
 	\\1&Aw	
 	\\Ou	
rK   c                   | j                   rt        d       | j                  }| j                  |z  }| j                  }| j
                  }t        | j                        D ]t  }|j                  |   dd|   j                         }| j                  j                  |j                  d|j                               | j                  j                  |d   |d   gg       | j                  j                  ||d|   g       | j                  j                  |dd|          | j                   j                  ||dz
  d|          ||dz  d|   }| j"                  j                  |       | j$                  j                  ||dz     g       ~~|j&                  |   }| j&                  j                  |j                  d|j                               w | j(                  dkD  r| j*                  }	| j,                  }
| j.                  }|j*                  d| dd|   j                         }t1        |      }||	d| t3        j4                  |dd|   |d   gf      | _        t1        | j6                        }| j6                  |	|||z    |j,                  dd |
dd || _        | j(                  | j*                  j8                  _        | j.                  | j,                  j8                  _        | j                   rt        d       d| _        | j<                  j?                          tA        | jB                        jE                          d| _!        d| _#        d	| _$        y)
z>Copy the data and delete the temporaries for sorting purposes.zCopying temporary data...Nr	   r   r   rU   zDeleting temporaries...FT)%r  r   r   r   r   r	  r   r   r   r   r
  r  r  r   r   r   r   r   r   r   r   r   r   r   r   r   r  r   r   r   rf  re  r   r   unlinkrc   r   )rZ   r   r  r  r	  rE   r   r  r   r   r   r:  rI  r   r  s                  rI   r$  zIndex.cleanup_temp,  s   <<-.NN	^^y(hht||$AjjmKiK0557GKKwq',,?@KKWR[ 9:;KKB01LL2/LLQ" 56bAgmm,GLL(LL!2 34kk!nGLL7<< @A# %( q IJ,,L||M\2;Y;?DDFHMM(0In}%NNHTrTNXb\N+KLDM$--(GAEImmg&=>MM!,JqM -D(,(9(9%)-):):&<<+,T%%' 
rK   c                   | j                   }| j                  x}	}
| j                  }t        j                  | j
                  |      }t        |      D ]  }||z   }||k(  r| j                  |z  }
t        |
      D ]M  }|||	z  |z      }||	z  }|||	z  z
  |z  }||z  }||z  }||k(  r||||z    ||||z    <|||||z   f   ||||z    O ||k(  r|d|
|z   |d|
|z   |||<    y)z)Get sorted & indices values in new order.r   N)r   r   r   r   r   r}   r   r   )rZ   newordersrc_disktmp_disklastrowr   r  r   r  r  ncs2self_nslicesr	  rE   nsrA  r   insincncs                       rI   get_neworderzIndex.get_neworderi  s+    ^^%%%d||hhT^^59wA!B\!((B.4[q3w{+SjS3Y",vV,&(/cBh(?CR"W%(0cC"Hn1D(ECR"W% ! \!'*;TBY'7$)$  #7  rK   c                   dddd}| j                   }|j                  }|j                  }|j                  }|j                  }|j
                  }|j                  }	| j                  }
| j                  }| j                  }||z  }|}|j                  ||         }| j                  dk(  xr | j                  dkD  }t        | j                        D ]  }| j                  ||z  z
  }||k  r|}|dk  r n||z  }|||z  ||z  |z    }|| j                  dz
  k(  r:|r8|dz  }| j                  |
z  }t!        j"                  || j$                  d| f      }|j'                  t(        	      }t+        ||z        }| j-                  ||||||| j.                         | j-                  ||||	||d
| j                           | j1                  d       y)z/Swap & reorder the different chunks in a block.r   r   r   )r  r  r  rW   r   r	   Nr[   uT)r	  )r	  r   r   rj  rk  r   r   r   r   r~   _f_get_childrX   r   r   r   r   r   r  r   argsortdefsortr   r  r   reorder_slices)rZ   rS  boundsnamesr	  r   r   
tmp_sortedtmp_indicesr   r   r  r  nsbncbncb2	boundsobjcan_cross_bboundsnblockremainingchunksr   r   ulsbounds_idxr  s                           rI   rO  zIndex.swap_chunks  s   
 

 hh**++[[
llLL	]]
^^Ci$$[%67	 LLA-G$2C2Ca2GDLL)F"llVc\9O$&qyckGv|fslT.ABF)).?1&&",s1C(DE ..g.6K#&F

 DLL>"7 *J 	%rK   c                   t        j                  ||gt         j                        | _        t        j                  |dz   ||j                  z   gt         j                        | _        t        j                  dt         j                        | _        |j                  | j                  | j
                  | j                  |       y)=Read a slice from the `where` dataset and put it in `buffer`.r	   rU   r   N)	r   r2  r8   startlr  stoplonesstepl_g_read_slice)rZ   r  nslicebufferr  s        rI   
read_slicezIndex.read_slice  s|    
 hh		:XXvz56;;+>?K
WW1BII6
DKKTZZHrK   c                   t        j                  ||gt         j                        | _        t        j                  |dz   ||j                  z   gt         j                        | _        t        j                  dt         j                        | _        | j
                  | j                  z
  }|j                  | j                  | j                  ||       y)z>Write a `slice` to the `where` dataset with the `buffer` data.r	   rU   r   N)	r   r2  r8   r  r  r  r  r  _g_write_slice)rZ   r  r  r  r  countls         rI   write_slicezIndex.write_slice  s     hh		:XXvz56;;+>?K
WW1BII6
dkk)T[[$**ffErK   c                *   t        j                  |gt         j                        }t        j                  ||j                  z   gt         j                        }t        j                  dgt         j                        }|j	                  ||||       y)r  r   r	   N)r   r2  r8   r  r  )rZ   r  r  r  r  r  r  s          rI   r   zIndex.read_slice_lr  sd     5'3%&++-.bii@!BII.FE5&9rK   c                *   t        j                  |gt         j                        }t        j                  ||j                  z   gt         j                        }t        j                  dgt         j                        }|j	                  ||||       y)>Write a slice from the `where` dataset with the `buffer` data.r   r	   N)r   r2  r8   r  r  )rZ   r  r  r  r  r  r  s          rI   write_slice_lrzIndex.write_slice_lr  sd     5'356;;./ryyA!BII.VUFF;rK   c                `    t        j                  dt        d       | j                  |||      S )r  zN'Index.write_sliceLR' is deprecated, please use 'Index.write_slice_lr' insteadrU   )
stacklevel)r!  r"  DeprecationWarningr  )rZ   r  r  r  s       rI   write_sliceLRzIndex.write_sliceLR  s2     	8		
 ""5&%88rK   c                T   | j                   }| j                  ||||d        | j                  ||||d        t        j                  ||       | j	                  ||dz
  |d|        | j	                  ||dz
  |d|        | j                  |dz
  |d|        ||d |d| ||d |d| y)z8Copy & reorder the slice in source to final destination.Nr	   )r}   r  r
   r   r  r6  )	rZ   r  r   r   rF  rG  r  r  r9  s	            rI   reorder_slicezIndex.reorder_slice  s     ^^
FGBCL9VXbc];  (3&1*gcrl;&1*hsm<6A:ws|4rs| "rK   c                   | j                   }| j                  }| j                  }|ddg   |j                  |<   ||d|   |j                  |<   |dd|   |j
                  ||z  |dz   |z   ||dz
  d|   |j                  ||z  |dz   |z   ||dz  d|   }||j                  ||z  |dz   |z   ||dz     |j                  |<   y)z%Update the caches for faster lookups.r   r   Nr	   rU   )	r   r   r	  r   r   r   r   r   r   )rZ   r  rF  r  r  r	  r  s          rI   r6  zIndex.update_caches   s    ^^hh$aW-

6$RVV_

69@BFSLFQJ##569@a29NFSLFQJ##56"'-R-(9@FSLFQJ##56%cQh/FrK   c                   | j                   }|j                  }|j                  }|r|j                  }|j                  }n|j                  }|j                  }| j
                  }| j                  }| j                  | j                  z  }| j                  }	| j                  }
| j                  }t        j                  |dz  | j                        }t        j                  |dz  t        j                  d| j                  z              }| j                  dk(  r| j                  |d|d|        | j                  |d|d|        d}t!        d|j"                        D ]  }| j%                  |||||||        |dkD  r| j                   j&                  }| j                   j(                  }|d||z    }||d }|d||z    }||d }| j+                  ||       | j+                  ||       t-        j.                  ||       | j1                  ||       | j1                  ||       t        j2                  |dd|   |d   gf      }||||t5        |      z    || _        | j9                  |||d|        | j9                  |||d|        | j;                  ||d|        yt!        |
      D ]  }||z  }| j                  |||d|        | j                  |||d|        ||z   }||	kD  r|	}|}t!        |dz   |      D ]  }| j%                  |||||||        | j9                  |||d|        | j9                  |||d|        | j;                  ||d|         y)	an  Reorder completely the index at slice level.

        This method has to maintain the locality of elements in the
        ambit of ``blocks``, i.e. an element of a ``block`` cannot be
        sent to another ``block`` during this reordering.  This is
        *critical* for ``light`` indexes to be able to use this.

        This version of reorder_slices is optimized in that *two*
        complete slices are taken at a time (including the last row
        slice) so as to sort them.  Then, each new slice that is read is
        put at the end of this two-slice buffer, while the previous one
        is moved to the beginning of the buffer.  This is in order to
        better reduce the entropy of the regular part (i.e. all except
        the last row) of the index.

        A secondary effect of this is that it takes at least *twice* of
        memory than a previous version of reorder_slices() that only
        reorders on a slice-by-slice basis.  However, as this is more
        efficient than the old version, one can configure the slicesize
        to be smaller, so the memory consumption is barely similar.

        rU   r   r-  rW   r   Nr	   r   )r	  r   r   rj  rk  r   r}   rz   r   r   r   r   r   r   rX   r  r   r   r  r   r   r   r
   r   r  r  r   r   r  r6  )rZ   r	  r   r   r  r  r  r9  r  r   r   r:  rF  rG  r  r   r   ssorted2rI  	sindices2	indiceslrr   nbr   lrbs                            rI   r  zIndex.reorder_slices0  sY   . hh**++J,,KJ++K^^^^nn.,,,,((((atzz:88"q&9M0NO<<1 OOJ73B<8OOKHSbM:F7==1"" 2 a HH--	!XX//
"#6R,%67#BC=$%8rL'89	%bcN	""9h7"":y9 ((9=##Ix8##J	:>>8DbD>HRL>*JK ,H)EF !)Wfgcrl;Wfhsm<vws|4 Gn Cx
D'#2,?T8CR=A Sj=!C#D1Hc2F&& "# 3   &'#2,?  &(3B-@""673B<87 %rK   c                d   | j                   }|j                  }|j                  }|j                  }|j                  }| j
                  }| j                  | j                  z  }|}	t        | j                        D ]  }
| j                  |
|z  z
  }||k  r|}	|	dk  r y
|dk(  r|j                  |
|z  |
|z  |	z   df   }n?|dk(  r|j                  |
|z  |
|z  |	z   df   }n|dk(  r|j                  |
|z  |
|z  |	z    }j                  t              }|t        j                   |	      k7  j#                         dz  }|dz  |	k  r| j$                  rt'        d	|
       |
|	z  }t        |	      D ]  }||   }||z   }||z   }||   ||<   ||   ||<   |j                  |   |j(                  |<   |j                  |   |j*                  |<   |j,                  |   |j.                  |<   ||z  }|dz   |z  }||z  }|dz   |z  }|j0                  || |j2                  || |j4                  || |j6                  || |j8                  || |j:                  ||  t        |	      D ]  }||z   }||   ||<   ||   ||<   |j(                  |   |j                  |<   |j*                  |   |j                  |<   |j.                  |   |j,                  |<   ||z  }|dz   |z  }|j2                  || |j0                  || |j6                  || |j4                  || |j:                  || |j8                  ||   y
)zSwap slices in a superblock.r	   r  r   r  r  r  rU   2   z$skipping reordering of superblock ->N)r	  r   r   rj  rk  r   ry   r}   r   r   r   r   r   r  r  r   r   sumr  r   rp  rq  r   rl  r   rm  r   rn  r   ro  )rZ   rS  r	  r   r   r  r  r  nssnss2sblockremainingslicesr   sranges_idxndiffr{  rE   r   oioidxrA  jnxjxjns                           rI   rP  zIndex.swap_slices  sK    hh**++[[
ll!!T^^3D--.F"llVc\9O$&qywFSL6C<$3F$F$IJFSL6C<$3F$F$IJ!Vc\FSL44GH ..g.6K BIIdO388:Q>ErzD  <<@&I$B4[!!n!VCx!(
2")$-B"%**T"2B#&;;t#4R "%**T"2BH1f^CZax3&%([[C%8Qr"%([[C%8Qr"%([[C%8Qr"' !* 4[!V(n)"o!$R

2"%,,r"2B!$R

2H1f^$'LL2$6Ab!$'LL2$6Ab!$'LL2$6Ab! !] /rK   c                   t        |d   |      rt        ||d         sJ | j                  }| j                  }| j                  }||z  }	|| j
                  k  rt        |j                  |   ||	      }
nt        | j                  ||	      dz
  }
|
dk\  sJ |
|z  }|| j
                  k  r*|t        |j                  ||||z   f   |      z  }||k  s1J ||z   }||kD  r|}|t        | j                  || |      z  }||k  sJ |dkD  sJ |S )z0Search a single item in a specific sorted slice.r   r	   )r!   r"   r   r}   r   r   r#   r   r   r   r   )rZ   r  itemr  limitsr  r  r9  r:  bstartnchunkposends                rI   r5  zIndex.search_item_lt  s/    F1It,dF1I1NNN^^^^((" DLL  f!5tVDF !f=AF{{ rkDLL ;u||FC#(N,BCTJJC"99(C\!";t}}S5t<<C,&&&Qww
rK   c           	        | j                   }|j                  dd }|j                  }|j                  }| j                  }| j
                  }	|	dkD  rNt        j                  | j                  d   | j                  d   g      }
t        j                  ||
gf      }|dz  }d}d}t        j                  |d      }|j                         }|j                         }t        |      D ]  }||df   }t        |dz   |      D ]  }||   }||k  sJ ||k(  r|| j                  k  r|||f   }n||   }||df   }||kD  rK|||z
  xx   dz  cc<   || j                  k  r||xx   ||z
  z  cc<   |||<   s||xx   |	|z
  z  cc<   |	||<   ||kD  s|||z
  xx   dz  cc<   | j                  |||||   |      }||z
  }||xx   |z  cc<   |||<   | j                  dk7  s|t        ||df         t        ||df         z
  z  } ! | j                  dk7  r't        |d         t        |d	         z
  }|dkD  r||z  }|rJ|d
k7  rEt!        d|z  |       t!        d||j#                                t!        d||j#                                |j#                         }| j$                  dk(  r.| j&                  j)                         r|dk(  | j*                  _        || _        |||fS )a  Compute some statistics about overlaping of slices in index.

        Returns
        -------
        noverlaps : int
            The total number of elements that overlaps in index.
        multiplicity : array of int
            The number of times that a concrete slice overlaps with any other.
        toverlap : float
            An ovelap index: the sum of the values in segment slices that
            overlaps divided by the entire range of values.  This index is only
            computed for numerical types.

        Nr   r   r	   r4  r   stringr   r	   r   r   initztoverlap (%s):zmultiplicity:
z
overlaps:
rW   )r}   r   r   r   r   r   r   r2  r   r  r3  r   r   r5  r   floatr   r  rX   rp   _iswritablerb   r   r   )rZ   r  rW  r  r9  r   r   r   r   r:  r;  soverlaptoverlapmultiplicityoverlapsr   rE   r@  rA  rB  rC  rD  r   nelemeranger   s                             rI   compute_overlaps_finegrainz Index.compute_overlaps_finegrain  s   " ^^a,,NN	,,((!xxq!14==3D EFH^^VhZ$89FqLGxxgV<$$&""$wAad|H1q5'*Qiby y"9t||#&q#vH(~H!!Q$<h& Q'1,'4<<' rCx/$&q	 |c'99$0q	( Q'1,'--xF1IsC  #IEQK5(K #F1IyyH, !E&A,$7%q!t:M$MM? +  H 99 6%=)E&,,??F z#f,w&("W,h7#\<3C3C3EF-8<<>:LLN	<<1!9!9!;#,>DMM "<22rK   c           	     ^   |j                   dd }| j                  }| j                  dkD  rNt        j                  | j
                  d   | j
                  d   g      }t        j                  ||gf      }|dz  }d}d}d}	t        j                  |d      }
t        |      D ]o  }t        |dz   |      D ][  }||df   ||df   kD  s|dz  }|
||z
  xx   dz  cc<   | j                  dk7  s8|t        ||df         t        ||df         z
  z  }] q | j                  dk7  r't        |d         t        |d	         z
  }|dkD  r||z  }	|rt        d
|z  ||	       t        |
       | j                  dk(  r.| j                  j                         r|dk(  | j                  _        || _        ||
|	fS )a  Compute some statistics about overlaping of slices in index.

        Returns
        -------
        noverlaps : int
            The total number of slices that overlaps in index.
        multiplicity : array of int
            The number of times that a concrete slice overlaps with any other.
        toverlap : float
            An ovelap index: the sum of the values in segment slices that
            overlaps divided by the entire range of values.  This index is only
            computed for numerical types.

        Nr   r   r	   r4  r   r  r  r  zoverlaps (%s):rW   )r   r   r   r   r2  r   r  r3  r   r   r  r   rX   rp   r  rb   r   r   )rZ   r  rW  r  r   r   r;  r   r  r  r  rE   rA  r  s                 rI   r   zIndex.compute_overlapsz  s   " a,,q xxq!14==3D EFH^^VhZ$89FqLG	xxgV<wA1q5'*!Q$<&A,.NI Q'1,'yyH, !E&A,$7%q!t:M$MM +   99 6%=)E&,,??F z#f,"W,iB,<<1!9!9!;#,>DMM "<22rK   c           	        | j                  |||      \  }}}||k\  r t        j                  d| j                        S |dk  r |}| j                  |z
  }| j                  |z
  }|dk(  r=| j
                  }| j                  }t        j                  ||z
  | j                        }n?| j                  }| j                  }t        j                  ||z
  d| j                  z        }| j                  }	||	z  }
||	z  }||	z  }||z
  }d}d}t        |
|dz         D ]j  }|	|z
  }||z   |kD  r||z
  }|dk  r nQ|| j                  k  r| j                  ||||||z    |       n| j                  |||||z    |       d}||z  }||z  }l |dd|   S )z;Return the sorted or indices values in the specified range.r   r   r   r-  r	   N)_process_ranger   r   r   r   r   r   r   r   rX   r}   r   r   r  r   )rZ   rR  r  r  stepr	  values	values_lrbuffer_r9  
nrow_startistart	nrow_stoptlenr  ilenr   blens                     rI   read_sorted_indiceszIndex.read_sorted_indices  s    #11%tDdD=88Atzz**!8CNNT)E>>C'D8[[FIhhte|4::>G\\FIhhte|54<<3GHG^^b[
BJ	e|*i!m4D;Dd{T!d{qydll"D'&6D="A6 ""wv> FdNFDLD! 5" vvrK   c                *    | j                  d|||      S )zReturn the sorted values of index in the specified range.

        The meaning of the start, stop and step arguments is the same as in
        :meth:`Table.read_sorted`.

        r   r  rZ   r  r  r  s       rI   read_sortedzIndex.read_sorted  s     ''%tDDrK   c                *    | j                  d|||      S )zReturn the indices values of index in the specified range.

        The meaning of the start, stop and step arguments is the same as in
        :meth:`Table.read_sorted`.

        r   r  r  s       rI   read_indiceszIndex.read_indices  s     ''	5$EErK   c                    ||t        |dz         }|d}nt        |      }|t        | j                        }nt        |      }|d}nt        |      }|||fS )z)Get a range specific for the index usage.r	   r   )r   r   r  s       rI   r  zIndex._process_range  sk     EAI&D=EUOE<DNN+DD>D<DD>DtT""rK   c                    t        |      rBt        j                  |      }|dk  r|| j                  z  }| j	                  ||dz   d      d   S t        |t              r1| j	                  |j                  |j                  |j                        S y)a  Return the indices values of index in the specified range.

        If key argument is an integer, the corresponding index is returned.
        If key is a slice, the range of indices determined by it is returned.
        A negative value of step in slice is supported, meaning that the
        results will be returned in reverse order.

        This method is equivalent to :meth:`Index.read_indices`.

        r   r	   N)
r   operatorindexr   r  
isinstanceslicer  r  r  )rZ   keys     rI   __getitem__zIndex.__getitem__  s|     #;..%CQwt~~%$$S#'15a88U#$$SYY#((CC $rK   c                    | j                   S re   )r   rY   s    rI   __len__zIndex.__len__1  s    ~~rK   c                T   | j                   j                  }| j                  | _        t	        |d   |d   d      | j                  _        t	        |d   |d   d      | j                  _        	 t	        |d   |d   d      | _        	 t	        |d   |d   d	      | _        	 t        j                  | j                  t        j                  
      | _        t        j                  | j                  t        j                  
      | _        | j                  j                  |        d| _        y)z<Clean the limits cache and resize starts and lengths arrays.BOUNDS_MAX_SLOTSBOUNDS_MAX_SIZEznon-opt types boundsLIMBOUNDS_MAX_SLOTSLIMBOUNDS_MAX_SIZEzbounding limitsSORTEDLR_MAX_SLOTSSORTEDLR_MAX_SIZEzlast row chunksr   FN)rp   paramsr   _sortedr%   boundscachelimboundscachesortedLRcacher   r   r   r3   r   r   _init_sorted_slicer   )rZ   r  s     rI   restorecachezIndex.restorecache4  s   $$ {{#.%&$%"$
 
 #.%&$%"#

	))()'(

 	+('(&'

	>hhTZZrxx@xxdjjA&&t,rK   c                |   t         r
t               }t         rt        d       | j                  r| j	                          |r|d   |d   kD  rd| j
                  dd d| j                  dd yd}| j                  j                  |      }|dk\  rZ| j                  j                  |      }d| j                  dd |D ])  \  }}}|| j
                  |<   || j                  |<   ||z   }+ |S | j                  }	| j                  dkD  r| j                  | j                  v r| j                  dk(  r |	j                  | }n| j                  dk(  r |	j                  | }nq| j                  dk(  r |	j                   | }nQ| j                  dk(  r |	j"                  | }n1| j                  d	k(  r |	j$                  | }n| j                  d
k(  r |	j&                  | }n| j                  dk(  r |	j&                  | }n| j                  dk(  r |	j(                  | }n| j                  dk(  r |	j*                  | }n| j                  dk(  r |	j,                  | }nv| j                  dk(  r |	j.                  | }nW| j                  dk(  r |	j0                  | }n8| j                  dk(  r |	j2                  | }nJ d       | j5                  ||	      }| j6                  dkD  r=| j9                  |      \  }}
|| j
                  d<   |
|z
  | j                  d<   ||
|z
  z  }| j                  j;                         rwg }t=        | j                        D ],  \  }}|dkD  s|j?                  || j
                  |   |f       . tA        |      dz  dz  dz   }| j                  jC                  |||       t         rt        d       |S )-Do a binary search in this index for an item.zEntering searchr   r	   Nr3   r4   float16r9   r:   float96float128r7   r8   r1   r2   r5   r6   zThis can't happen!r   rW   rU   zExiting search)"r   r   r&   r   r   r   r   r  getslotgetitemr   r   r   r   _search_bin_na_i_search_bin_na_ll_search_bin_na_e_search_bin_na_f_search_bin_na_d_search_bin_na_g_search_bin_na_ui_search_bin_na_ull_search_bin_na_b_search_bin_na_s_search_bin_na_ub_search_bin_na_ussearch_scalarr   search_last_rowcouldenablecacher@   r
  r   setitem)rZ   r  r   r  nslotstartlengthsnrow2r  lengthr   r  r   r  s                rI   searchzIndex.search[  s   7D($/?? tAwa(DKKNDLLO##++D1A:..66u=LDLLO(4$uf%*E"&,U#f} )5 K++<<!yyD111 99'3733T:DYY')4744d;DYY)+3733T:DYY)+3733T:DYY)+3733T:DYY)+3733T:DYY*,3733T:DYY(*4744d;DYY(*5755t<DYY&(3733T:DYY')3733T:DYY')4744d;DYY(*4744d;D6"665))$8q  006MUD#DKKO#e|DLLD5L D//1 L )$,, 7fA: ''t{{4/@&(IJ !8 |$q(1,q0D''lDA'.rK   c                    d}t        | j                        D ]@  }|j                  ||      \  }}|| j                  |<   ||z
  | j                  |<   |||z
  z  }B |S )r  r   )r   r   _search_binr   r   )rZ   r  r   r  rE   r  r  s          rI   r  zIndex.search_scalar  se     t||$A#//48MUD"DKKN"UlDLLOD5L D	 %
 rK   c                   |\  }}| j                   }|d   |d   }}|dd }| j                  j                  }| j                  }	| j                  }
| j
                  | j                  z  }d}t        ||      rt        ||      rt        ||      }|	j                  |      }|dk\  r|	j                  |      }nb||z  }||dz   z  }||
kD  r|
}| j                  j                  | j                  ||      }|	j                  ||j!                         ||z
  |z         t        ||      }|||z  z  }n|
}nd}t#        ||      rt%        ||      rt'        ||      }||k7  r|	j                  |      }|dk\  r|	j                  |      }nb||z  }||dz   z  }||
kD  r|
}| j                  j                  | j                  ||      }|	j                  ||j!                         ||z
  |z         t'        |      }|||z  z  }||fS |
}||fS d}||fS )zSearch the last row.r   r   r	   )r   r   r   r  r   r   r   r   r"   r#   r  r  r   _read_sorted_slicer   r  r   r    r!   r$   )rZ   r  item1item2r   b0b1r   r   sorted_lr_cachehir   r  r  chunkbeginr  r  nchunk2r  s                       rI   r  zIndex.search_last_row  sJ    u==!hrlB!B::&&,,^^t~~5
r"E2&$VU3'//7A:+33E:E&/E$
3CRx  MM<<UCE
 $++

sU{h.F $E51f,,Er"E2&&vu5f$+33G<Ez / 7 7 > *W 4(GaK88"$C $ @ @ KK! (//#UZZ\C%K83K $E51
W,,
 t}  t} Dt}rK   c                   t         r
t               }t         rt        d       | j                  }| j                  }| j
                  }| j                  }| j                  }| j                  |z  }t        j                  | j                  |z        }t        j                  |d      }	| j                  }
| j                  dz
  |
z  dz   }| j                  | j                   z   |
z  }d||dk  <   | j"                  }t%        | j&                        D ]  }||   }||   }||kD  st        j(                  ||z
  d|z        }||k  r|j+                  ||||       n| j,                  j+                  |||       |dk(  r2|t        j.                  |      j1                  |j2                        z  }nP|dk(  r(|j1                  d	      }t5        ||z  |z        }||z  }n#|dk(  r|j1                  d	      }||z  |z  }||z  }d
|	|<    | j6                  }||k7  r| j                  }t        j                  ||z        }t        j                  |d      }||z  }|	j9                         d   }||z  j1                  d	      }t        j                  |dz   |z        j1                  d	      }t;        ||      D ]
  \  }}d
|||  |}	t         rt        d       |	S )z3Compute a map with the interesting chunks in index.zEntering get_chunkmaprf   r   r	   r   r-  rW   rU   r4  TzExiting get_chunkmap)r   r   r&   r}   r~   r   r   rX   rz   r   r   r   r   r3  r   r   r   r   r   r   r   _read_index_slicer   r   r   r   r   r   rQ  zip)rZ   r   r9  r  r   r   rX   bucketsinblockr   chunkmapr   r   stopsr   r  r  r  r   r  r   r   tnchunks	tchunkmapratios                           rI   get_chunkmapzIndex.get_chunkmap	  s   7D.5^^,,,,,,'1))DNNW4588'8NN	++/Y.2t||+y8vz,,DJJ'F6NE=De|hhTE\IG#--feT3GNN44UD#Fa<BJJw/66syyAAC\**V,C &C->!ABF6MC\**V,C$rkg5F6MC $+ (. ((l"Iyy\!9:Hxv>Il*E""$Q'CEk))&1FGGS1W-.55f=E"651t(,	%%  2 H-t4rK   c                   t        |      dv sJ t        |      dv sJ t        |      t        |      k(  sJ | j                  }|j                  j                  }|j                  }t        |      dk(  r|d   dv sJ |d   }|d   }|dk(  rt        ||d      t        |d||      f}|S |dk(  rt        ||d      |f}|S |d	k(  rt        |d||      t        ||d      f}|S |d
k(  r|t        ||d      f}|S |dk(  r||f}S t        |      dk(  ru|d   dv r|d   dv sJ |\  }	}
|	|
kD  ry|dk(  rt        |	d||      t        |
d||      f}|S |dk(  r|	t        |
d||      f}|S |dk(  rt        |	d||      |
f}|S |dk(  r|	|
f}S )zReturn the lookup range.)r	   rU   r	   r   )ltleeqgegtr5  r   )signr6  r9  r8  r7  rU   )r9  r8  )r5  r6   )r9  r5  )r8  r5  )r9  r6  )r8  r6  )r   rv   r   r   r   r   r   )rZ   opsr  rv   coldtyper   limitoprange_loweruppers              rI   get_lookup_rangezIndex.get_lookup_rangeQ	  s'    3x6!!!6{f$$$3x3v;&&&<<$$$$v;!q6;;;;1IEQBTzHhR8eR8<H A t!(H2>F> = teR8<HhR8: 3 t8"!EF0 / t, ) [Aq6\)c!f.DDD!LE5u}l"eR8<eR8<  $5"h!IJ  $#E2xBEJ  $rK   c                $    t         |   d       y)zRemove this Index object.TN)r   	_f_remove)rZ   	recursiver   s     rI   rE  zIndex._f_remove	  s     	$rK   c                   g }| j                   j                  r| j                   j                  r|j                  d       | j                   j                  r|j                  d       |j                  | j                   j
                   d| j                   j                   d       d| j                   d| j                   dj                  |       d| j                   S )z5Return the string representation of the Index object.r/   
bitshufflerM  rN  zIndex(z, z	).is_csi=)
r^   r-   r/   r
  rH  r.   r   r[   rB   r   )rZ   r^   s     rI   __str__zIndex.__str__	  s     <<!!||##y)||&&|,NNdll2231T\\5K5K4LANOT]]O2yyk$))G,-Yt{{mE	
rK   c                   | j                   j                   d| j                  j                   }dj	                  g | j                   d| d| j
                   d| j                   d| j                   d| j                   d| j                   d	| j                   d
| j                   d| j                   d| j                   d| j                   d| j                  d| j                    d| j"                   d| j$                   d| j&                   d| j(                   d| j*                         }|S )z:Return the string representation including also  metainfo.z.cols. z (Index for column z)
  optlevel := z
  kind := z
  filters := z
  is_csi := z
  nelements := z
  chunksize := z
  slicesize := z
  blocksize := z
  superblocksize := z
  dirty := z
  byteorder := z
    sorted := z
    indices := z
    ranges := z
    bounds := z
    sortedLR := z
    indicesLR := )rg   ro   rv   pathnamerB   r   r[   r^   r   r   r   r}   rz   ry   rc   r   r   r   r   r   r   r   )rZ   	cpathnameretstrs      rI   __repr__zIndex.__repr__	  s   zz--.fT[[5I5I4JK	% %d&&' %': %9+ % F %}}o%%99+%% ll^%% [[M	%	%
 %
 % % % % % % % (()%*% JJ<%% !%"% {{m%% ~%% {{m%% {{m%%  !%  !%" ..!#%$ rK   )returnz0Literal['ultralight', 'light', 'medium', 'full'])rP  r   rP  rf   )rc   rf   rP  None)rP  r)   rP  r*   )rP  r   )rP  tuple[int, int])
NrK  NNNNr   NNT)r   r   rF   strr   zAtom | Noner   rU  r[   z7Literal['ultralight', 'light', 'medium', 'full'] | Noner   
int | Noner^   zFilters | Noner   
str | Noner   r   r   rW  r   z tuple[int, int, int, int] | Noner   rf   rP  rR  rP  rR  )r   list[np.ndarray]r   r   r   r   rP  z"tuple[int, np.ndarray, np.ndarray])r   
np.ndarrayr  r   rP  rZ  )F)r   rY  r  rf   rP  rR  )r  rf   rP  rR  re   )rR  zLiteral['chunks', 'slices']rS  z)Literal['start', 'stop', 'median'] | NonerP  rf   )ru  rZ  rv  r'   rw  r'   rx  r   r   r   r  r   r   znpt.DTypeLikerP  rR  )r  )rS  z"Literal['start', 'stop', 'median']rP  rR  r   )
r  r'   r  r   r  rZ  r  r   rP  rR  )r  r'   r  rZ  r  r   rP  rR  )r  r   r   r'   r   r'   rF  rZ  rG  rZ  r  r'   r  r'   rP  rR  )r  r   rF  rZ  rP  rR  )r  r(   r  r  r  r   r  tuple[float, float]r  r   rP  r   )r  r(   rW  rU  r  rf   rP  ztuple[int, np.ndarray, float])
rR  zLiteral['indices', 'sorted']r  rV  r  rV  r  rV  rP  rZ  )NNN)r  rV  r  rV  r  rV  rP  rZ  )r  rV  r  rV  r  rV  rP  ztuple[int, int, int])r  zint | slicerP  zint | np.ndarray)r  r[  rP  r   )r  ztuple[float | int, float | int]r   r   rP  r   )r  r[  rP  rT  )rP  rZ  )r<  ztuple[str] | tuple[str, str]r  z"tuple[float] | tuple[float, float]rP  r[  )rF  rf   rP  rR  )rP  rU  )D__name__
__module____qualname____doc__
_c_classidpropertyr[   r^   rc   setterrv   rg   r{   r~   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r
  r  r+  r   r  r   r  r$  r  rO  r  r  r   r  r  r  r6  r  rP  r5  r  r   r  r  r  r  r  r  r   r  r  r  r3  rC  rE  rI  rO  __classcell__)r   s   @rI   rM   rM   q   s   3j J
 
   6 6 \\	$ 	$ 
 
   5 5 0 0 0 0     0 0 0 0 * * 
 
 8 8 # #* ( (  4 !HL#"&" $7;M@M@ M@ 	M@
 M@ FM@ M@  M@ M@ M@ M@ 5M@ M@ 
M@^eNR$R,/R<?R	+Rh&>/B 6;/1$/1.2/1	/1bJXwMx ;?/)/ 8/ 
	/bu
nY
v;z*#*# *# 	*#
 *# *# *# *# 
*#Z :B<&6<&	<&~ KLII$'I1;IDGI	I KLFF$'F1;FDGF	F >?::$.:7::	: >?<<$.<7:<	< >?
9
9$.
97:
9	
9&& & 	&
 & & & & 
&20 v9r :BI76I7	I7b && & 	&
 $& & 
&P]3]3),]37;]3	&]3~=3=3),=37;=3	&=3~0*0 0 	0
 0 
0h !	EE E 	E
 
E  !	FF F 	F
 
F##'1#9C#	#*D*% N`F3>H	IV9v7)7 37 
	7r 
rK   rM   c                      e Zd ZdZdZddZy)IndexesDescGzIndexes descriptor for groups.DINDEXc                R    t        j                  d| j                  z  t               y )Nzthe number of indexed columns on a single description group is exceeding the recommended maximum (%d); be ready to see PyTables asking for *lots* of memory and possibly slow I/Or!  r"  _v_max_group_widthr   rY   s    rI   _g_width_warningzIndexesDescG._g_width_warning	  (    $ '+&=&=> 	
rK   NrX  )r\  r]  r^  r_  r`  rj  r;  rK   rI   re  re  	  s    (J
rK   re  c                      e Zd ZdZdZed
d       Zej                  dd       Zej                  dd       ZddZ	ddZ
edd       Zy	)IndexesTableGzTable indexes.TINDEXc                V    d| j                   vrt        S | j                   j                  S )z!Return True if auto-index is set.
AUTO_INDEX)rb   default_auto_indexrp  rY   s    rI   autozIndexesTableG.auto	  s&     t}},%%}}'''rK   c                8    t        |      | j                  _        y re   )rf   rb   rp  )rZ   rr  s     rI   rr  zIndexesTableG.auto	  s    #': rK   c                    | j                   `y re   )rb   rp  rY   s    rI   rr  zIndexesTableG.auto	  s    MM$rK   c                R    t        j                  d| j                  z  t               y )Nzthe number of indexed columns on a single table is exceeding the recommended maximum (%d); be ready to see PyTables asking for *lots* of memory and possibly slow I/Orh  rY   s    rI   rj  zIndexesTableG._g_width_warning	  rk  rK   c                B    |j                  d      st        d|z        y )Nr=   z1names of index groups must start with ``_i_``: %s)rA   
ValueError)rZ   rF   s     rI   _g_check_namezIndexesTableG._g_check_name	  s(    u%CdJ  &rK   c                    | j                   j                  d      }|j                         dd }dj                  |      }t	        ||      }| j
                  j                  |      }|S )zBAccessor for the `Table` object of this `IndexesTableG` container.r<   r>   N)ro   r?   r   rB   r   rp   rq   )rZ   rD   	tablenameparentpathnamerG   rg   s         rI   rg   zIndexesTableG.table	  s]       &&s+IIKO	%!.)<&&}5rK   NrQ  )rr  rf   rP  rR  rX  )rF   rU  rP  rR  rS  )r\  r]  r^  r_  r`  ra  rr  rb  deleterrj  rx  rg   r;  rK   rI   rm  rm  	  sj    J( ( 
[[. . 
\\% %
  rK   rm  c                      e Zd ZdZdZy)OldIndexz4This is meant to hide indexes of PyTables 1.x files.CINDEXN)r\  r]  r^  r_  r`  r;  rK   rI   r~  r~  	  s
    >JrK   r~  )rC   rU  rP  ztuple[str, str])Ur_  
__future__r   rd  r   r   r  rb  r!  timer   r   r   r1  typingr   r   pathlibr   numpyr   numpy.typingnptrK  r
   r   r   r   leafr   r   r   pathr   groupr   utilsr   r   r   carrayr   earrayr   indexesr   r   r   idxutilsr   r   r   r   r   
exceptionsr   utilsextensionr   r    r!   r"   r#   r$   lrucacheextensionr%   r   r   r   r&   r2  r'   r(   rg   r)   r*   r  rq  default_index_filtersr   max32rJ   rM   re  rm  r~  r;  rK   rI   <module>r     s   & " 	 
     & ) )            - -   9 9  +  +
 	 	

! "
 
    %   	(C%NE#3#9#9 C%LJ
>5 
+NE +\~u rK   