+dA\dZddlmZddlmZmZmZddlmZGdde Z Gd d e Z Gd d Z dd lmZn #e$reZYnwxYwGddZGddeZGddZGddeZGdde eeeZGddeZdS)z( Base classes for Data Loader interface )dump)DictSerialiseMixinmake_path_posixpretty_describe) make_cachesc.eZdZdZfdZdZdZxZS)SchemazHolds details of data description for any type of data-source This should always be pickleable, so that it can be sent from a server to a client, and contain all information needed to recreate a RemoteSource on the client. c tt|jdi|dD] }||vrd||< d|vrd|d<d|vri|d<dSdS)z^ Parameters ---------- kwargs: typically include dtype, shape )dtypeshapeN npartitionsrextra_metadata)superr __init__)selfkwargsfield __class__s 2lib/python3.11/site-packages/intake/source/base.pyrzSchema.__init__s %fd$..v...' # #ED  #"U  $ $"#D  4 ' (%'D! " " " ( (cNd|j|j|jS)Nz2 dtype: {} shape: {} metadata: {})formatr rrrs r__repr__zSchema.__repr__)s3RYYZ^ZdfjfprvsFGG Grc||SNr)ritems r __getattr__zSchema.__getattr__,s Dzr)__name__ __module__ __qualname____doc__rrr! __classcell__rs@rr r sc((((( GGGrr ceZdZdS)NoEntryN)r"r#r$rrrr)r)0sDrr)cneZdZdZdZedZdZedZej dZdZ dS) CacheMixinz*Allows "old style" caching for Data SourceNc$d|jDS)Ncg|] }|j Sr) _cache_dir).0cs r z)CacheMixin.cache_dirs..;s111 111r)cachers r cache_dirszCacheMixin.cache_dirs9s11dj1111rcB|jD]}t||_dSr)r2rr.)r cache_dirr0s r set_cache_dirzCacheMixin.set_cache_dir=s. 6 6A*955ALL 6 6rc|jSt|j|jd|jdd|j|_|jS)Nr2 catalog_dir)catdirstorage_options)_cacher namemetadatagetr:rs rr2zCacheMixin.cacheAsj ; b%di1B1B71K1KTXTaTeTefsuyTzTzMQMabbbDK{rc||_dSr)r;)rcshs rr2zCacheMixin.cacheGs  rc`t|jdkrgSfd|jDS)Nrc:g|]}|Sr)load)r/r0urlpaths rr1z)CacheMixin._get_cache..Ns#444Aw444r)lenr2rrDs `r _get_cachezCacheMixin._get_cacheKs< tz??a  9 44444444r) r"r#r$r%r;propertyr3r6r2setterrGrrrr+r+4s44 F 22X2666X  \\55555rr+)hvPlotcjeZdZdZedZedZedZedZdS)HoloviewsMixinz#Adds plotting and GUI to DataSourcecRt|jdiS)z"List custom associated quick-plotsplots)listr=r>rs rrNzHoloviewsMixin.plotsZs$DM%%gr22333rc|jjS)z1Source GUI, with parameter selection and plotting)entryguirs rrRzHoloviewsMixin.gui_sz~rcltturtd|jdi}|jdi}|D]}d|vrt |d|d<||d<|jdi}t|fd|i|S)at Plot API accessor This property exposes both predefined plots (described in the source metadata) and general-purpose plotting via the hvPlot library. Supported containers are: array, dataframe and xarray, To display in a notebook, be sure to run ``intake.output_notebook()`` first. The set of plots defined for this source can be found by >>> source.plots ["plot1", "plot2"] and to display one of these: >>> source.plot.plot1() To create new plot types and supply custom configuration, use one of the methods of ``hvplot.hvPlot``: >>> source.plot.line(x="fieldX", y="fieldY") The full set of arguments that can be passed, and the types of plot they refer to, can be found in the doc and attributes of ``hvplot.HoloViewsConverter``. Once you have found a suitable plot, you may wish to update the plots definitions of the source. Simply add the ``plotname=`` optional argument (this will overwrite any existing plot of that name). The source's YAML representation will include the new plot, and it could be saved into a catalog with this new definition. >>> source.plot.line(plotname="new", x="fieldX", y="fieldY"); >>> source.plots ["plot1", "plot2", "new"] zThe intake plotting API requires hvplot.hvplot may be installed with: `conda install -c pyviz hvplot` or `pip install hvplot`.plotfieldsrangerN custom_plots)rJobject ImportErrorr=r>valuestuple HVplotCapture)rr=rUattrsrNs rrTzHoloviewsMixin.plotdsP V  ] =$$VR00""8R00]]__ 7 7E% 7!&uW~!6!6g# !!'2..TBBBBBBrc|jS)z/ alias for ``DataSource.plot`` )rTrs rhvplotzHoloviewsMixin.hvplots yrN) r"r#r$r%rHrNrRrTr_rrrrLrLWs-- 44X4X2C2CX2ChXrrLc(eZdZdZddfd ZxZS)r\z)rrms r is_persistedzPersistMixin.is_persisteds1------}  66%??rNc ddlm}ddlm}d|jvrt d|.t |ttfst d||}|j | |fi|}|j ||j in|j d |j|_|||||S) aSave data from this source to local persistent storage Parameters ---------- ttl: numeric, optional Time to live in seconds. If provided, the original source will be accessed and a new persisted version written transparently when more than ``ttl`` seconds have passed since the old persisted version was written. kargs: passed to the _persist method on the base container. rrjr) PersistStorer|z4Cannot persist a source taken from the persist storeNzOCannot persist using a time to live that is non-numeric. User-provided ttl was )ttlcat)rorkrprr= ValueError isinstanceintfloat_exportgetdirupdater __getstate__r<add)rrrrkrrmouts rpersistzPersistMixin.persists '&&&&&444444 T] * XVWW W  y:cC<#@#@ ywruwwxx x dl5<<--8888 Cdh0cDHLaLaLcLcddeee9 ((4..#&&& rr) r"r#r$r%rr staticmethodrxrHrzr}rrrrrgrgs::'''""\"''X'@@X@ rrgceZdZdZdZdZdZdZdZdZ dZ dZ dZ dZ dZdZddZdZdZdZd Zd Zd Zd Zd ZedZedZdZdZdZdZdZ dZ!dZ"edZ#dZ$e$xZ%Z&dZ'dZ(dZ)dZ*dZ+dZ,dS) DataSourceBaseaSAn object which can produce data This is the base class for all Intake plugins, including catalogs and remote (server) data objects. To produce a new plugin commonly involves subclassing this definition and overriding some or all of the methods. This class is not useful in itself, most methods raise NotImplemented. NFrc|pi|_t|jtr||jdi}||_dS)Nr:)r=rdict_captured_init_kwargsr>r:)rr:r=s rrzDataSourceBase.__init__sR B dmT * * T T"8<<=NPRSSO.rct)z3Subclasses should return an instance of base.Schemartrs r _get_schemazDataSourceBase._get_schema!!rct)zSubclasses should return a container object for this partition This function will never be called with an out-of-range value for i. rtris r_get_partitionzDataSourceBase._get_partition s "!rct)z*Subclasses should close all open resourcesrtrs r_closezDataSourceBase._closerrc|jr||_|jj|_|jj|_|jj|_|j|jjdSdS)zload metadata only if neededN)_schemarr rrr=rrrs r_load_metadatazDataSourceBase._load_metadataso < >++--DL+DJ+DJ#|7D  M !< = = = = =  > >rc Vddl}|j}|d|jpi}|t t||j j |j d|j |j |jpd||dii}|S)Nrr=sources)driver descriptionr=re)inspectrcopypopr=rrzip signaturer parameters_captured_init_argsr< classnamer)rrrmetadatas r_yamlzDataSourceBase._yamls+0022zz*dm44: d3w00??JDLdeeffgggDI$.QUQaQgeguyDJ(K(KLM rcL|}t|dS)zReturn YAML representation of this data-source The output may be roughly appropriate for inclusion in a YAML catalog. This is a best-effort implementation Fdefault_flow_style)rr)rrs ryamlzDataSourceBase.yaml&s$ zz||DU3333rcddlm}|d}t|d}||t |ddd|jiid d S) z9Display the entry as a rich object in an IPython session.r)displayrFr)zapplication/yamlz text/plainzapplication/jsonrootT)r=rawN)IPython.displayrrrrr<)rrrcontentss r_ipython_display_z DataSourceBase._ipython_display_/s++++++zz||I&777X_X=V=VWWcux~AEAJxKcLRV W W W W W Wrc*|Sr)rrs rrzDataSourceBase.__repr__7syy{{rcdSz1The base class does not interact with persistenceFrrs rr}zDataSourceBase.is_persisted: urcdSrrrs rrzz!DataSourceBase.has_been_persisted?rrc|S)z,The base class does not interact with cachesrrFs rrGzDataSourceBase._get_cacheDsrcx|t|j|j|j|jS)z1Open resource and populate the source attributes.)r rrr=)rrr rrr=rs rdiscoverzDataSourceBase.discoverHs6 $*DJDDT_c_lmmmmrc^|jr |jdkr|dSt)z2Load entire dataset into a container and return itrr)partition_accessrrrurs rreadzDataSourceBase.readNs6$ &(8A(= &&&q)) )% %rc#K|t|jD]}||VdS)z7Return iterator over container fragments of data sourceN)rrVrrrs r read_chunkedzDataSourceBase.read_chunkedUsY t'(( ) )A%%a(( ( ( ( ( ) )rc||dks ||jkrtd|z||S)zReturn a part of the data corresponding to i-th partition. By default, assumes i should be an integer between zero and npartitions; override for more complex indexing schemes. rz%d is out of range)rr IndexErrorrrs rread_partitionzDataSourceBase.read_partition[sY  q5 7A)) 71A566 6""1%%%rct)z,Return a dask container for this data sourcertrs rto_daskzDataSourceBase.to_daskgrrct)aProvide an equivalent data object in Apache Spark The mapping of python-oriented data containers to Spark ones will be imperfect, and only a small number of drivers are expected to be able to produce Spark objects. The standard arguments may b translated, unsupported or ignored, depending on the specific driver. This method requires the package intake-spark rtrs rto_sparkzDataSourceBase.to_sparkks "!rc<|jtd|jS)Nz(Source was not made from a catalog entry)_entryr)rs rrQzDataSourceBase.entryws$ ; FDEE E{rc LjUfdjD}||jdi|}j|_|Sj}||t ji|S)aCreate a new instance of this source with altered arguments Enables the picking of options and re-evaluating templates from any user-parameters associated with this source, or overriding any of the init arguments. Returns a new data source instance. The instance will be recreated from the original entry definition in a catalog **if** this source was originally created from a catalog. Nc.i|]\}}|jv||Sr)_passed_kwargs)r/kvrs r z0DataSourceBase.configure_new..s,bbb41adNaIab!Qbbbrr)rritemsrrtyper)rrkwobjs` r configure_newzDataSourceBase.configure_new}s ; ?bbbb4#=#C#C#E#EbbbB IIf   $+####CCJJ+0022B IIf   4::t7>2>> >rc4|jS)zDescription from the entry spec)rQdescribers rrzDataSourceBase.describesz""$$$rc.|dS)z7Close open resources corresponding to this data source.N)rrs rclosezDataSourceBase.closes rc.||Sr)rrs r __enter__zDataSourceBase.__enter__s  rc.|dSr)r)rexc_type exc_value tracebacks r__exit__zDataSourceBase.__exit__s rc |j|fi|S)aSave this data for sharing with other people Creates a copy of the data in a format appropriate for its container, in the location specified (which can be remote, e.g., s3). Returns the resultant source object, so that you can, for instance, add it to a catalog (``catalog.add(source)``) or get its YAML representation (``.yaml()``). )r)rrwrs rexportzDataSourceBase.exportst|D++F+++rc ddl}ddlm}ddlm}||jj}||fd|i|}|j|_||j||j |||d}||_|j |_ |S)Nrrjr) container_maprw) timestamporiginal_metadataoriginal_source original_namer|persist_kwargs) timerork containerrrxrr=rr<) rrwrrrkrmethodrr=s rrzDataSourceBase._exports &&&&&&------t~.7fT/////*!%#0022!Y$HTNN$     9 r)NN)-r"r#r$r%r<versionrrrr rrr on_serverrrrrrrrrrrrrHr}rzrGrrrrrrrQrrdr>rrrrrrrrrrrs DGIK E EKGI C F////""""""""">>>444WWWXXnnn &&&))) & & &""" " " "X ???,#"Hs%%%  , , ,rrceZdZdZdS) DataSourcezA Data Source will all optional functionality When subclassed, child classes will have the base data source functionality, plus caching, plotting and persistence abilities. N)r"r#r$r%rrrrrs  DrrceZdZdZedZejdZedZejdZedZdS) PatternMixinzEHelper class to provide file-name parsing abilities to a driver classcNt|dr|jStd)N_path_as_patternzr s)HHHHHHHHHHT>     n   55555555: FFFGGGGGGGGT 1 1 1 1 1F 1 1 166666666r_____'___D     ^\>   ,,,,,6,,,,,sA AA