
    |Le1;                       d Z ddlmZ ddlZ ej                    ZddlZddlZddlZddlZddl	Z	ddl
Z
ddlZddlZddlmZ ddlZddlZddlZddlZddlmZ ddlmZ ddlmZ dadada  G d d	          Z! e!            Z"d
hZ# G d de$          Z%dBdZ& e$d dD                       Z'dBdZ(d e'd         d<   d e'd         d<   d Z)d e'd         d<   d e'd
         d<   d e'd         d<   d e'd         d<   dBdZ*d e'd         d <   d! e'd         d <   d" e'd         d <   d# Z+d$ e'd
         d <   d% e'd         d <   d& e'd         d <    e$d' dD                       Z,d( e,d         d<   d) e,d         d<   d* Z-d+ e,d         d<   d, e,d
         d<   d- e,d         d<   d. e,d         d<   d/ e,d         d <   d0 e,d         d <   d1 e,d         d <   d2 e,d
         d <   d3 e,d         d <   d4 e,d         d <   dCd7Z.d8 Z/d9a0 e1d:           e1d;          d<fd=Z2d> Z3d? Z4d@ Z5e6dAk    r  ej7         e5ej8                             dS dS )Daw  
Simple test of read and write times for columnar data formats:
  python filetimes.py <filepath> [pandas|dask [hdf5base [xcolumn [ycolumn] [categories...]]]]

Test files may be generated starting from any file format supported by Pandas:
  python -c "import filetimes ; filetimes.base='<hdf5base>' ; filetimes.categories=['<cat1>','<cat2>']; filetimes.timed_write('<file>')"
    )annotationsN)export_image)transfer_functions)distributedFc                  l    e Zd Zd\  ZZZdZg ZdZdZ	dZ
dZ eddd	          Z ej                    ZdS )

Parameters)dataxypandasiOߑ   Ng   Ј BhiveF)file_scheme	has_nullswrite_index)__name__
__module____qualname__baser
   r   dftype
categories	chunksize	cat_widthcolumns	cachesizedict	parq_optsmultiprocessing	cpu_count	n_workers     =lib/python3.11/site-packages/datashader/examples/filetimes.pyr   r   $   s]        HD1FJIIGIdvEJJJI'o'))IIIr"   r   parqc                      e Zd ZdZdS )KwargszWUsed to distinguish between dictionary argument values, and
    keyword-arguments.
    N)r   r   r   __doc__r!   r"   r#   r&   r&   5   s          	Dr"   r&   c                   t          |          }i }|rDt          |d         t                    r)|                                }|                    |           t
          rd                    d |D                       }d                    d |                                D                       }t          d	                    | j
        ||rd|z   nd          d           t          j                    } | |i |}	||t          vr<i }
t          j        d
k    rd|
d<   t          j        D ]} |	|         j        di |
|	|<   t          j        dk    rit"          rbt$          /t$                              |	          }	t)          j        |	           n,t
          rt          dd           |	                                }	t          j                    }||z
  |	fS )zBenchmark when "fn" function gets called on "args" tuple.
    "args" may have a Kwargs instance at the end.
    If "filetype" is provided, it may be used to convert columns to
    categorical dtypes after reading (the "loading" is assumed).
    z, c                    g | ]B}t          |d           r!t          |                                          nt          |          CS )head)hasattrstrr+   ).0posargs     r#   
<listcomp>zbenchmark.<locals>.<listcomp>I   sG    &|&|&|jpWVV=T=T'es6;;=='9'9'9Z]^dZeZe&|&|&|r"   c                @    g | ]\  }}d                      ||          S )z{}={})format)r.   kvs      r#   r0   zbenchmark.<locals>.<listcomp>J   s*    %U%U%UsqgnnQ&:&:%U%U%Ur"   zDEBUG: {}({}{}) TflushNr   Fcopycategorydaskz#DEBUG: Force-loading Dask dataframe)r9   )list
isinstancer&   popupdateDEBUGjoinitemsprintr2   r   timefiletypes_storing_categoriespr   r   astypeDD_FORCE_LOADDASK_CLIENTpersistr   wait)fnargsfiletypeposargskwargslastargprintable_posargsprintable_kwargsstartresoptscends                r#   	benchmarkrX   ;   s    4jjGF :gbk622 ++--g G II&|&|t{&|&|&|}}99%U%Ufllnn%U%U%UVV&&r{4E`pGxtL\G\G\vxyy  BF  	G  	G  	G  	G IKKE
"g
 
 
 C 777Dx8##"V\ 8 8$s1v}77$77A 8v-& "))#.. %%%% M?tLLLLkkmm
)++Cu9c>r"   c                .    g | ]}|t                      fS r!   r   r.   fs     r#   r0   r0   m   s     XXXQa[XXXr"   )r$   snappy.parqgz.parqfeatherh5csvc                    t           j                            |           rt          j        | |          S |                     dd          }t          j        ||          S )Nusecols.csv*.csv)ospathisfileddread_csvreplace)filepathrd   filepath_exprs      r#   read_csv_daskro   o   sW    	w~~h 6{8W5555$$VW55M;}g6666r"   c                X    t          t          | t          |j                  f|          S Nrc   )rX   ro   r&   r   rm   rE   rM   s      r#   <lambda>rs   v   s5    yYacirsr{c|c|c|X}  @H  @I  @I r"   ra   r:   c           	     z    t          t          j        | |j        t	          |j        |j                  f|          S )N)r   r   )rX   rj   read_hdfr   r&   r   r   rr   s      r#   rs   rs   w   s_    yW_abagioz{  {F  PQ  PY  jZ  jZ  jZ  W[  ]e  @f  @f r"   r`   c                    t          j        | t          j                  }t	          j        |t          j                  S )Nr   )npartitions)r_   read_dataframerE   r   rj   from_pandasr    rm   dfs     r#   read_feather_daskr}   x   s0    		!)	<	<	<B>"!+6666r"   c                0    t          t          | f|          S N)rX   r}   rr   s      r#   rs   rs   {   s    YGX[cZego=p=p r"   r_   c                d    t          t          j        | t          d|j                  f|          S NF)indexr   rX   rj   read_parquetr&   r   rr   s      r#   rs   rs   |   R    y[cekrw  BC  BK  fL  fL  fL  [M  OW  @X  @X r"   c                d    t          t          j        | t          d|j                  f|          S r   r   rr   s      r#   rs   rs   }   r   r"   r^   c                d    t          t          j        | t          d|j                  f|          S r   r   rr   s      r#   rs   rs   ~   r   r"   r]   c                   t           j                            |           rt          j        |           S |                     dd          }t          j        |          }t          j        fd|D                       S )Nrc   re   rf   c              3  D   K   | ]}t          j        |           V  dS )rc   N)pdrk   )r.   r\   rd   s     r#   	<genexpr>z"read_csv_pandas.<locals>.<genexpr>   s2      II!bk!W555IIIIIIr"   )rg   rh   ri   r   rk   rl   globconcat)rm   rd   rn   	filepathss    `  r#   read_csv_pandasr      sw    	w~~h 6{8W5555$$VW55M	-((I9IIIIyIIIJJJr"   c                X    t          t          | t          |j                  f|          S rq   )rX   r   r&   r   rr   s      r#   rs   rs      s3    iZbdjsts|d}d}d}Y~  AI  ?J  ?J r"   r   c                n    t          t          j        | |j        t	          |j                  f|          S )Nrw   )rX   r   ru   r   r&   r   rr   s      r#   rs   rs      sO    iV^`a`fhnwx  xA  iB  iB  iB  VC  EM  ?N  ?N r"   c                :    t          t          j        | f|          S r   )rX   r_   ry   rr   s      r#   rs   rs      s    iH^ai`kmu>v>v r"   c                N    t          j        |                                           S r   )fpParquetFile	to_pandas)rm   s    r#   read_parq_pandasr      s    >(##--///r"   c                0    t          t          | f|          S r   rX   r   rr   s      r#   rs   rs          iHX[cZego>p>p r"   c                0    t          t          | f|          S r   r   rr   s      r#   rs   rs      r   r"   c                0    t          t          | f|          S r   r   rr   s      r#   rs   rs      r   r"   c                .    g | ]}|t                      fS r!   rZ   r[   s     r#   r0   r0      s     YYYaq$&&kYYYr"   c                t    t          | j        |                    dd          t          d          f          S )Nre   rf   Fr   )rX   to_csvrl   r&   r|   rm   rE   s      r#   rs   rs      sQ    )BIPXP`P`aghoPpPprx  @E  sF  sF  sF  PG  ;H  ;H r"   c                :    t          | j        ||j        f          S r   )rX   to_hdfr   r   s      r#   rs   rs      s    )BIPXZ[Z`Oa:b:b r"   c                P    t          j        |                                |           S r   )r_   write_dataframecomputer{   s     r#   write_feather_daskr      s    "2::<<:::r"   c                0    t          t          || f          S r   )rX   r   r   s      r#   rs   rs      s    	BTW_acVd8e8e r"   c                :    t          t          j        || f          S r   )rX   rj   
to_parquetr   s      r#   rs   rs      s    )BMT\^`Sa:b:b r"   c                X    t          t          j        || t          d          f          S )NSNAPPYcompressionrX   rj   r   r&   r   s      r#   rs   rs      s.    )BMT\^`bhu}b~b~b~S  ;A  ;A r"   c                X    t          t          j        || t          d          f          S )NGZIPr   r   r   s      r#   rs   rs      s(    )BMT\^`bhu{b|b|b|S}:~:~ r"   c                L    t          | j        |t          d          f          S )NFr   )rX   r   r&   r   s      r#   rs   rs      s%    )BIPXZ`glZmZmZmOn:o:o r"   c                X    t          | j        |t          |j        d          f          S )Ntable)keyr2   )rX   r   r&   r   r   s      r#   rs   rs      s+    )BIPXZ`efekt{Z|Z|Z|O}:~:~ r"   c                :    t          t          j        | |f          S r   )rX   r_   r   r   s      r#   rs   rs      s    )GD[^`bj]k:l:l r"   c           
     Z    t          t          j        || t          di |j        f          S )Nr!   rX   r   writer&   r   r   s      r#   rs   rs      s-    )BHxY[]c]r]rfgfq]r]rNs:t:t r"   c           
     ^    t          t          j        || t          dddi|j        f          S )Nr   r   r!   r   r   s      r#   rs   rs      sU    )BHxY[]c  ^G  ^Gpv  ^Gz{  {F  ^G  ^G  OH  ;I  ;I r"   c           
     ^    t          t          j        || t          dddi|j        f          S )Nr   r   r!   r   r   s      r#   rs   rs      sU    )BHxY[]c  ^I  ^Ipx  ^I|}  }H  ^I  ^I  OJ  ;K  ;K r"   doubletimesc                   |dv sJ |t           _        t          | |          \  }}t                                          D ]<}t
          j                            |           \  }}t
          j                            |          \  }	}
|t
          j        j	        z   |	z   dz   |z   }t
          j        
                    |          r%t          d                    |          d           |                    d          d         }|t          vrmt           j        D ]`}|dk    r5||         j        dk    r$||         j                            d	          ||<   =||                             t                     ||<   a|d
k    rC|j        D ];}||         j        dk    r(||                             t(          j                  ||<   <t          |                             |d          }|&t          d                    ||          d           n; |||t                     \  }}t          d                    |||          d           |t          vr-t           j        D ] }||                             d          ||<   !>dS )ztAccepts any file with a dataframe readable by the given dataframe type, and writes it out as a variety of file types)singler   .z{:28} (keeping existing)Tr6   r)   r$   objectutf8r   float64Nz"{:28} {:7} Operation not supportedz{:28} {:7} {:05.2f}r9   )rE   r   
timed_readr   keysrg   rh   splitsplitextsepexistsrB   r2   rD   r   dtyper-   encoderF   r   npfloat32get)rm   r   fsizeoutput_directoryr|   durationext	directoryfilenamebasename	extensionfnamerM   rV   colnamecoderT   s                    r#   timed_writer      sj   (((((AH8F++KBxzz|| 3 3W]]844	( g..x88) ,X5c9#=7>>%   	3,33E::$GGGGGYYs^^B'H;;; 0 0A6))bekX.E.E ei..v661 ell3//1   !z E EG'{(I55&(k&8&8&D&D7:>>&..D|:AA%OOW[\\\\\ $Ra 0 0#+225IIQUVVVV;;; 3 3AQ%,,z22BqEE?3 3r"   c                p   t           j                            |           \  }}|dd          }|                    d          d         }t          |                             |d           }|dS t          j        gt          j        gz   t          j	        z   t          _
         || t          |          \  }}||fS )Nr   r   r)   )Nr)   )rg   rh   r   r   readr   rE   r
   r   r   r   )rm   r   r   r   rM   r   r   r|   s           r#   r   r      s    '**844Hi!""I__S!!"%H	?vd++D|zseQSEk!,&AI48,,LHbx<r"   )NNi  g     h@Tc                v   t          j                     }t          j        ||t          d         t          d                   }|                    | t
          j        t
          j                  }t          j                     }|r|j        |j	        fat          t          j        |          |d          }	|	||z
  fS )Nr   r   )x_rangey_ranger   )export_path)rC   dsCanvasCACHED_RANGESpointsrE   r
   r   r   r   r   tfshade)
r|   rm   
plot_widthplot_heightcache_rangesrS   cvsaggrW   imgs
             r#   	timed_aggr      s    IKKE
)J]15E}]^O_
`
`
`C
**Rac
"
"C
)++C 3ck2
rx}}X#
>
>
>CE	>r"   c                   d}t           j                            |           rt           j                            |           S |                     d          rOt          j        |                     dd                    D ]$}|t           j                            |          z  }%|S t          j        |           D ]M\  }}}|D ]D}t           j                            ||          }|t           j                            |          z  }EN|S )Nr   ra   re   rf   )	rg   rh   ri   getsizeendswithr   rl   walkr@   )rh   total	csv_fpathdirpathdirnames	filenamesr\   r   s           r#   get_sizer      s    E 
w~~d wt$$$	u		 4<<#@#@AA 	0 	0IRW__Y///EE )+ ) )$9 	) 	)Agq))BRW__R(((EE	) Lr"   c                 N    t          j        t           j                  j        dz  S )N    .A)resource	getrusageRUSAGE_SELF	ru_maxrssr!   r"   r#   get_proc_memr      s    h233=CCr"   c                >   t          j        t          t           j                  }|                    d           |                    d           |                    d           |                    d           |                    d           |                    dd	           |                    d
dd           |                    ddd d                    t          t          j                                       |                    ddd           |                    ddd           |	                    | dd                    }|j
        |j        rt          dd           nx|j
        dk    r0ddlm}  |t          j                  }|                                 n|j
        dk    rda|j        r)t          d                    |j
                  d           |j        dk    rR|j        rKt%          j        t          j        d           }t%          j        |          a|j        rt          d!           n7|j        dk    r|j        rt/          d"          |j        rt          d#           |j        }t2          j                            |          \  }}|j        t          _        |j        t          _        |j        t          _        |j        t          _        |j        t          _        |j        a t@          r0t          d$                    tC                                d           tE          |t          j                  \  }	}
|	7|
d%k    r/t          d&                    |t          j                  d           dS t@          r0t          d'                    tC                                d           tG          |	|d(d(|j$         )          \  }}t@          r|	%                    d*          }t          j        dk    r|&                                }t          d+|d           |'                                }t          d,                    |d-z            d           |	j(        D ]0}t          d.                    ||	|         j)                             1t          d/                    tC                                d           tG          |	||j$         )          \  }}t@          r0t          d0                    tC                                d           tU          |          }tU          |d1z             }tW          j+                    }t          d2                    |t          j        |
|z   |
|||||tX          z
  	  	        d           dS )3N)epilogformatter_classrm   r   r   r
   r   r   +)nargsz--debug
store_truez-Enable increased verbosity and DEBUG messages)actionhelpz--cache)rI   cacheyzEnable caching: "persist" causes Dask dataframes to force loading into memory; "cachey" uses dask.cache.Cache with a cachesize of {}. Caching is disabled by default)choicesdefaultr  z--distributedzOEnable the distributed scheduler instead of the threaded, which is the default.z--recalc-rangeszdTell datashader to recalculate the ranges on each aggregation, instead of caching them (by default).r   zDEBUG: Cache disabledTr6   r  r   )CacherI   zDEBUG: Cache "{}" mode enabledr:   )r    threads_per_workerz)DEBUG: "distributed" scheduler is enabledzR--distributed argument is only available with the dask dataframe type (not pandas)z&DEBUG: "threaded" scheduler is enabledz(DEBUG: Memory usage (before read):	{} MBr)   z#{:28} {:6}  Operation not supportedz'DEBUG: Memory usage (after read):	{} MB   )r   )deepzDEBUG:zDEBUG: DataFrame size:			{} MBr   zDEBUG: column "{}" dtype: {}z'DEBUG: Memory usage (after agg1):	{} MBz'DEBUG: Memory usage (after agg2):	{} MBz.pngzq{:28} {:6}  Aggregate1:{:06.2f} ({:06.2f}+{:06.2f})  Aggregate2:{:06.2f}  In:{:011d}  Out:{:011d}  Total:{:06.2f})-argparseArgumentParserr'   RawTextHelpFormatteradd_argumentr2   intrE   r   
parse_argscachedebugrB   
dask.cacher
  registerrG   r   r   LocalClusterr    ClientrH   
ValueErrorrm   rg   rh   r   r   r
   r   r   r?   r   r   r   recalc_rangesmemory_usager   sumr   r   r   rC   global_start)argvparserrL   r
  r  local_clusterrm   r   r   r|   loadtimer   aggtime1	mem_usagemem_usage_totalr   aggtime2in_sizeout_size
global_ends                       r#   mainr*    s    $GXEbcccF

###
!!!



C000
	,=lmmm
	+@$  V|  VC  VC  DG  HI  HS  DT  DT  VU  VU  V  V  V
  DU  V  V  V
),  Fl  m  m  mT!""X&&Dz: 	7)6666:!!((((((E!+&&ENNZ9$$ M: 	S299$*EETRRRR{f!1#01;[\]]]!(77: 	?=>>>;&  T%5 qrrr: 	<:;;;}H'**844HiKAHIAFFACFACOALJE ^9@@PPX\]]]]Xqx00KBx	zr>>7>>xRRZ^____q ]8??OOW[\\\\R1t?Q;QSSSLC 	]OOO..	8v!))++Ih	....#--//18839NOOW[\\\\z 	U 	UG077GARSSTTTT8??OOW[\\\\R4;M7MOOOLC ]8??OOW[\\\\!!G((HJ	
}6(AHhx&78XW^`hjt  vB  kB  C  C  KOP P P P 1r"   __main__r   )r   r   )9r'   
__future__r   rC   r  rg   os.pathsysr   r  r   r   r   r   dask.dataframe	dataframerj   numpyr   
datashaderr   r_   fastparquetr   datashader.utilsr   r   r   r:   r   r?   rG   rH   r   rE   rD   r   r&   rX   r   ro   r}   r   r   r   r   r   r   r   r  r   r   r   r*  r   exitr  r!   r"   r#   <module>r6     s    # " " " " " ty{{ B B B B B B B B B B B B B B B B B B B B B B B B B B B B                        ) ) ) ) ) ) / / / / / /       		* 	* 	* 	* 	* 	* 	* 	* *,, &x 	 	 	 	 	T 	 	 	. . . .d tXX"WXXXYY7 7 7 7 #I  #IUf  #f  #fT
f 7 7 7 !q pYf  #X  #XVf  #X  #XYf  #X  #X] f K K K K "J  "JUX  "N  "NT
X !v!vYX 0 0 0!p!pVX !p!pYX !p!p] X  	YY#XYYYZZ $H  $Hev #b#bdv ; ; ;!e!ei v #b#bfv  $A  $Am v #~#~i v #o#oex  #~#~dx  #l#li x  #t#tfx   $I  $Ii x   $K  $Km x  &3 &3 &3 &3R    '*s3xxSS__[_ 	 	 	 	  (D D DU U Up zCHTT#(^^ r"   