+dKddlZddlZddlZddlZddlZddlZddlZddlZddlmZddl m Z ddl m Z ddl mZmZejdZdZeZGdd eZdd ZGd d eZGddeZGddeZGddeZGddejjZeeeedZddifdZ dS)N)datetime)md5)conf) is_notebookmake_path_posixintakec^ddlm}||}|d}|dvrGtj|d|d}n;|dkr5tj|}|dd}t|S) zUtility for cleaning up paths.r)infer_storage_optionsprotocol)httphttpsz{}://file:) fsspec.utilsr ospathnormpathreplaceformatr)rr storage_optionr s 3lib/python3.11/site-packages/intake/source/cache.py sanitize_pathrs222222**400Nj)H$$%w W^^H-E-Er J JKK V %w%%||C$$ 4  creZdZdZdZddifdZdZdZdZddZ d Z dd Z d Z dd Z dZdZdZdZdS) BaseCachea Provides utilities for managing cached data files. Providers of caching functionality should derive from this, and appear as entries in ``registry``. The principle methods to override are ``_make_files()`` and ``_load()`` and ``_from_metadata()``. i@KLNc||_||_t|p td}|dkr&|t dt j|d}||_||_t|_ dS)a Parameters ---------- driver: str Name of the plugin that can load catalog entry spec: dict Specification for caching the data source. cache_dir: str or None Explicit location of cache root directory catdir: str or None Directory containing the catalog from which this spec was made cache_dircatdirNz?cache_dir="catdir" only allowed when loadedfrom a catalog file. intake_cache) _driver_specrr TypeError posixpathjoin _cache_dir_storage_options CacheMetadata _metadata)selfdriverspecrrstorage_optionscds r__init__zBaseCache.__init__<s|  Y;${*; < < > 8 f deee77B /&rctj|jstj|jtj|jr't d|jdS)Nz-Path for cache directory exists as a file: {})rrexistsr&makedirsisfile Exceptionrr*s r_ensure_cache_dirzBaseCache._ensure_cache_dirUsrw~~do.. ) K ( ( ( 7>>$/ * * hNUUVZVeffgg g h hrcddl}t|}d|jvr/dt|jdz}|j|d|}t j|j||dS)Nrregexz%srz/\)rerr"subr$r%r&lstrip)r* cache_subdirurlpathr9rr8s r _munge_pathzBaseCache._munge_path[sw W%% dj  +M$*W*=>>>E26%T**D~do|T[[=O=OPPPrc tttj||jdd|jf S)Nr8r) rstrrrbasenamer"getr!encode hexdigestr*r=s r_hashzBaseCache._hashfs[3((114:>>'23N3NPTP\]^^eegghhrrtttrcX|||}|||}tj|}tj|s>|ds)|dstj||S)Nzhttps://zhttp://)rFr>rrdirnamer1 startswithr2)r*r=subdir cache_pathrHs r_pathzBaseCache._pathis  )ZZ((F%%fg66 '//*--w~~g&& %&&z22 %g6H6H6S6S % G$$$rctj||d}|j||dS)N)created original_pathrK)rnow isoformatr)update)r*r=rOrKmetadatas r _log_metadatazBaseCache._log_metadatausB'|~~7799Mistt gx00000rc Ztjddr|gS||ntjdd|_||}|t d|Dr/||\}}||||||}|S)aW Downloads data from a given url, generates a hashed filename, logs metadata, and caches it locally. Parameters ---------- urlpath: str, location of data May be a local path, or remote path if including a protocol specifier such as ``'s3://'``. May include glob wildcards. output: bool Whether to show progress bars; turn off for testing Returns ------- List of local cache_paths to be opened instead of the remote file(s). If caching is disable, the urlpath is returned. cache_disabledFNcache_download_progressTc3VK|]$}tj| V%dSN)rrr1).0cs r z!BaseCache.load..s3%Q%Q"'..*;*;&;%Q%Q%Q%Q%Q%Qr)rrBoutput_from_metadataany _make_files_load)r*r=r]kwargs cache_pathsfiles_in files_outs rloadzBaseCache.loadys& 8$e , , 9  &affAZ\`8a8a ))'22  5#%Q%Q[%Q%Q%Q"Q"Q 5"&"2"27";"; Hi JJxG 4 4 4))'22 rcL||}| d|DSdS)/Return set of local URLs if files already existNcg|] }|d S)rK)rZes r z,BaseCache._from_metadata..s000AlO000r) get_metadatar*r=mds rr^zBaseCache._from_metadatas;   w ' '  100R000 0 1 1rTc ddl}g}g}t||D]7\}} | j} || | |kr)tj| st d|jt d|t d| |r| ||j| | t} || || |j |j 9|j||S)zDownload a set of filesrNCaching file: {}Original path: {} Cached at: {})daskziprappendrr3loggerdebugrrTdelayed _download blocksizer]compute) r*rdrer=metartoutoutnamesfile_infile_outrKddowns rrazBaseCache._loadsJ !$Xy!9!9 R R GX!J OOJ ' ' 'W$ 7>>*-- R /66w|DDEEE 077@@AAA _33J??@@@J&&w jIII Y// 55(DNDKPPQQQ crc t)z$Make OpenFiles for all input/outputs)NotImplementedErrorr*r=rbs rr`zBaseCache._make_filess!!rc6|j|S)a6 Parameters ---------- urlpath: str, location of data May be a local path, or remote path if including a protocol specifier such as ``'s3://'``. May include glob wildcards. Returns ------- Metadata (dict) about a given urlpath. )r)rBrEs rrmzBaseCache.get_metadatas~!!'***rc|j|g}|D]4} tj|d#tt f$rY1wxYw tj|d}tj|dS#tt f$r!t d|zYdSwxYw)a Clears cache and metadata for a given urlpath. Parameters ---------- urlpath: str, location of data May be a local path, or remote path if including a protocol specifier such as ``'s3://'``. May include glob wildcards. rKz$Failed to remove cache directory: %sN) r)poprremoveOSErrorIOErrorrrHrmdirrwrx)r*r= cache_entries cache_entryfns r clear_cachezBaseCache.clear_caches**7B77 (  K  +l34444W%     F\!:;;B HRLLLLL! F F F LL?"D E E E E E E Fs!<AA9B.CCcx|jD]}||tj|jsdStj|jD]} tj |j|}tj|rtj |tj |rtj |#ttf$r1}t t%|Yd}~d}~wwxYwdS)z0 Clears all cache and metadata. N)r)keysrrrisdirr&listdirr$r%shutilrmtreer3rrrrwwarningr@)r*r=rJrrks r clear_allzBaseCache.clear_alls)~**,, & &G   W % % % %w}}T_--  Fj11 ' 'F '^DOV<<7==$$&M"%%%7>>"%%"IbMMMW% ' ' 's1vv&&&&&&&& ' ' 's4BC55D7'D22D7rYT)__name__ __module__ __qualname____doc__r{r/r6r>rFrLrTrfr^rar`rmrrrjrrrr0sI,0DRT))))2hhh Q Q Quuu    111<111 4""" + + +FFF.'''''rrFc  tj5tjd|r4 ddlm}n,#t $rt dd}YnwxYwtr2 ddl }n,#t $rt dd}YnwxYw |j |j }| }nH#t$r;}t d|d}d }Yd}~nd}~wwxYw|ru|sqt!d D])} | t"vrt"| | } n*||d zd|| t&j |j d d } nd}t d|j |5} |5} d }|rW| |r|nd}| ||r%| t1|d z|Wdddn #1swxYwYdddn #1swxYwY|r | | j| jz | n4#t8$r'}t d|zYd}~nd}~wwxYwt"| n#t"| wxYwddddS#1swxYwYdS)z2Read from input and write to output file in blocksignorer)tqdmzHCache progress bar requires tqdm to be installed: conda/pip install tqdmFNzbCache progress bar in a notebook requires ipywidgets to be installed: conda/pip install ipywidgetsz%File system error requesting size: {}Tdig?z {n}/|/{l_bar})totalleavedisablepositiondesc mininterval bar_formatz Caching {}ztqdm exception: %s)warningscatch_warningsfilterwarningstqdm.autonotebookr ImportErrorrwwarnr ipywidgetsfssizer ValueErrorrxrrangedisplayaddrrAreadwriterRlenrncloser4r)rrr{r]rr file_size pbar_disablederrir~pbarf1f2datarks rrzrzs@  " ":$:$)))   2222222    ijjj  ==  !!!!!    DEEE  !  55I )MMM ! ! ! LL@GGLL M M MI MMMMMM !    sA' At#u,) ))(-88 #/ \((66777  8 8R877 #A99rBBDHHTNNN8 CII$6777 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8  $ $ DJ/000  7 7 7 1A566666666 7s####s####u:$:$:$:$:$:$:$:$:$:$:$:$:$:$:$:$:$:$sK83K8&AK8AK8.A32K83&BK8BK8 "CK8 D 1D>K8DB.K86H89AH! H8!H% %H8(H% )H8, K88H< <K8?H< K86I=<K = J.J)$K )J..K 1K8 K((K88K<?K<c(eZdZdZdZfdZxZS) FileCachezCache specific set of files Input is a single file URL, URL with glob characters or list of URLs. Output is a specific set of local files. c 6ddlmt|tt fr)d|n||dfij}fd|D}||fS)Nr open_filesrrbcng|]1}|jgdfijd2SwbrrLrr'rZfrr*rJs rrlz)FileCache._make_files..EJrrrdeZZAFF!;!; .]rrc3^K|](}tj|jV)dSrY)rrrHrZrs rr\z'DirCache._make_files.._s2>>BGOOAF++>>>>>>r)rrr6rFr"rextendr%setrurrr2r4rv)r*r=rbrrdrre files_in2 files_out2pathsfinfoutrrJs` @@rr`zDirCache._make_filesTs%%%%%%    G$$ 7#q%!)$$ I IA OOJJsxx SEAI0E'F'FGG H H H Hrrrrrriqrrr "B: >>X>>>>>Xy11 ( (ICx5  (K **** D  %%%!!$''''*$$sC77 DDc`||}|||gSdS)rhN)rmrLrns rr^zDirCache._from_metadataks<   w ' '  )JJw''( ( ) )rN)rrrrr`r^rjrrrrLs<%%%.)))))rrc*eZdZdZfdZddZxZS)CompressedCachea;Cache files extracted from downloaded compressed source For one or more remote compressed files, downloads to local temporary dir and extracts all contained files to local cache. Input is URL(s) (including globs) pointing to remote compressed files, plus optional ``decomp``, which is "infer" by default (guess from file extension) or one of the key strings in ``intake.source.decompress.decomp``. Optional ``regex_filter`` parameter is used to load only the extracted files that match the pattern. Output is the list of extracted files. c ddl}|ddlm|||_|dfi|j}fd|D}tt| |||d||fS)Nrrrc g|]b}ttjtj|jgddcSr)rrrr%rA)rZrdrs rrlz/CompressedCache._make_files..s\yyyklZZaAQAQRSRXAYAY1Z1Z![![ \^bccdefyyyrF)r}) tempfilemkdtemprrr6_urlpathr'rrra) r*r=rbrrdrerrrs @@rr`zCompressedCache._make_files~s     %%%%%%     :gtEEt/DEEyyyyypxyyy  ot$$**8Ye*TTT""rTc ddlm}||} tj|n#t t f$rYnwxYwd|D}g}t||D]\} } d|jvr|jddkr|jd} n| drd} n| ds| d rd } nt| d s| d rd } nG| drd} n/| drd} n| drd} | |vrtd| z|| | |} ttj |jddj| } | D]}t d| t d| jt d||r!||j| j|||ǐ|S)Nr)decompcg|] }|j Srj)rrs rrlz)CompressedCache._load..s+++A+++rrinferz.zipruz.tar.gzz.tgztgzz.tar.bz2z.tbztbzz.tartarz.gzgzz.bz2bzzUnknown compression for "%s" regex_filterz.*rqrrrs) decompressrrLrr2rrrur"endswithrfilterr9compilerBsearchrwrxrrrTrrv)r*rdrer=r}rrJfilesr~rorigrout2out3rs rrazCompressedCache._loads&&&&&&G$$  K    !    D +++++5(++  GAt4:% $*X*>'*I Jx(F## I&& !**V*<*< J'' 1::f+=+= F## E"" F##  E !?!!CDDD6!9Q''D"*TZ^^ND%I%IJJQSWXXD   /66q99::: 077 BBCCC _33B77888E&&t}diDDD 2   s2AAr)rrrrr`rarrs@rrrrsV   # # # # #%%%%%%%%rrc eZdZdZdZddZdS)DATCacheaUse the DAT protocol to replicate data For details of the protocol, see https://docs.datproject.org/ The executable ``dat`` must be available. Since in this case, it is not possible to access the remote files directly, this cache mechanism takes no parameters. The expectation is that the url passed by the driver is of the form: :: dat:///file_pattern where the file pattern will typically be a glob string like "\*.json". c .|dS)N)NN)r6rs rr`zDATCache._make_filess    zrTcddl}ddlm}tj|j||}tj|\}} dd||dg} | | |j n0#ttf$rtdwxYwtj|| } |r*|| D] } |||| jdSdS)Nrrdatclonez --no-watch)stdoutzCalling DAT failed) subprocessrrrrr%r&rFsplitcallPIPErrrwinforT) r*___r=r}rrrr partcmdnewpathofs rrazDATCache._loads)%%%%%%w||DOTZZ-@-@AAGMM'** TgsD,7  OOC O 8 8 8 8!    KK, - - -  ',,tT**  > j)) > >""7GRW==== > > > >s -B -B7Nr)rrrrr`rarjrrrrsA >>>>>>rrcVeZdZdZdZdZdZdZdZdZ dZ d Z d Z dd Z d Zd S)r(z_ Utility class for managing persistent metadata stored in the Intake config directory. cddlm}tjt |jd|_tj |j}tj |stj |tj |jrHt|j5}tj||_ddddS#1swxYwYdSi|_dS)Nr)configzcache_metadata.json)rrr$r%rconfdirrLrrrHr1r2r3openjsonrfr))r*argsrbrrrs rr/zCacheMetadata.__init__s!!!!!!^OFN$C$CEZ[[ GOODJ ' 'w~~a    KNNN 7>>$* % % dj!! .Q!%1 . . . . . . . . . . . . . . . . . . DNNNsC))C-0C-cB|j||SrY)r)__keytransform__r*keys r __getitem__zCacheMetadata.__getitem__s~d33C8899rch||j||<|dSrYr)r!_save)r*r#values r __setitem__zCacheMetadata.__setitem__s-5:t,,S112 rcd|j||=|dSrYr&r"s r __delitem__zCacheMetadata.__delitem__s+ N40055 6 rc*t|jSrY)iterr)r5s r__iter__zCacheMetadata.__iter__sDN###rc*t|jSrY)rr)r5s r__len__zCacheMetadata.__len__s4>"""rcht|ttfrd|}|S)Nr)rrrr%r"s rr!zCacheMetadata.__keytransform__s, cD%= ) ) ((3--C rc||}|j|g}||||j|<|dSrY)r!r)rBrvr')r*r#rentriess rrRzCacheMetadata.updates[##C((.$$S"--{###%s rct|jd5}tj|j|ddddS#1swxYwYdS)Nw)rrLrdumpr))r*rs rr'zCacheMetadata._save s $*c " " )a Idna ( ( ( ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) )s>AANcd|j||}||SrY)r)rr')r*r#defaultitems rrzCacheMetadata.pops+~!!#w//  rcNt|jSrY)rr)rr5s rrzCacheMetadata.keyssDN''))***rrY)rrrrr/r$r)r+r.r0r!rRr'rrrjrrr(r(s    :::$$$### ))) +++++rr()rdir compressedr c |gSg}|D]p}d|vr$|dtvrt|d|t|dt|||||q|S)a Creates Cache objects from the cache_specs provided in the catalog yaml file Parameters ---------- driver: str Name of the plugin that can load catalog entry specs: list Specification for caching the data source. Ntype)rrr-)registry IndexErrorrvrBr)r+specsrrr-r~r,s r make_cachesrBs  CMM T> +d6l(: +T&\** * 8<<V i88f`i|KLLL M M M M Jr)F)! collectionsrloggingrr$r9rrrhashlibr intake.configr intake.utilsrr getLoggerrwrrrobjectrrzrrrrabcMutableMappingr(r?rBrjrrrLs/   55555555  8 $ $!!!( #%%C'C'C'C'C'C'C'C'L<$<$<$<$~///// ///.#)#)#)#)#)y#)#)#)L>>>>>i>>>B&>&>&>&>&>y&>&>&>R:+:+:+:+:+KO2:+:+:+zhoV^ _ _'+dBr