U ef@sddlmZddlmZddlZddZGdddZdd Zd d Z d d Z ddZ d#ddZ ddZ d$ddZd%ddZddZGdddeZd&ddZd'd!d"ZdS)() quote_plus)ceilNcCst|ddS)N.z%2E) _quote_plusreplace)srl/mounts/lovelace/software/anaconda3/envs/qiime2-amplicon-2023.9/lib/python3.8/site-packages/redbiom/admin.pyrsrc@sLeZdZdZdddddZdZiZedd d Zed d Z ed dZ dS) ScriptManagerz>Static singleton for managing Lua scripts in the Redis backenda local indices = {} local kid = nil -- for each index and identifier (like python's enumerate) for position, name in ipairs(ARGV) do kid = redis.call('HGET', KEYS[1], name) -- if an identifier was not observed, add it if not kid then kid = redis.call('HINCRBY', KEYS[1], 'current_id', 1) - 1 redis.call('HSET', KEYS[1], name, kid) redis.call('HSET', KEYS[1] .. '-inverted', kid, name) end -- store store the mapping for return indices[position] = tonumber(kid) end return cjson.encode(indices)a -- Redis has a compile time stack limit for Lua calls -- so rather than recompiling with an arbitrary limit, -- we're going to instead chunk calls where there are a -- large number of arguments. The default is 8000 for the -- stack size, so we'll use 7900 to be close without -- going over -- https://stackoverflow.com/a/39959618/19741 local call_in_chunks = function (command, key, args) local step = 7900 for i = 1, #args, step do redis.call(command, key, unpack(args, i, math.min(i + step - 1, #args))) end end -- Lua does not have a natural split, for various reasons -- outlined in the URL below, so we need to do this -- manually. We'll split on "|" which should be safe -- as the values sent are only ever expected to be integers -- http://lua-users.org/wiki/SplitJoin for idx, arg in ipairs(ARGV) do local items = {} for item in string.gmatch(arg, "([^|]+)") do table.insert(items, item) end call_in_chunks('LPUSH', KEYS[idx], items) end return redis.status_reply("OK")a local context = ARGV[1] local key = ARGV[2] local result = {} local formedkey = context .. ':' .. 'feature' .. ':' .. key local items = redis.call('LRANGE', formedkey, '0', '-1') -- adapted from https://gist.github.com/klovadis/5170446 local resultkey local ii = context .. ':' .. 'sample' .. '-index-inverted' for idx, v in ipairs(items) do if idx % 2 == 1 then -- it is likely possible to issue a HMGET resultkey = redis.call('HGET', ii, v) else result[resultkey] = tonumber(v) end end return cjson.encode(result)a local context = ARGV[1] local key = ARGV[2] local result = {} local formedkey = context .. ':' .. 'sample' .. ':' .. key local items = redis.call('LRANGE', formedkey, '0', '-1') -- adapted from https://gist.github.com/klovadis/5170446 local resultkey local ii = context .. ':' .. 'feature' .. '-index-inverted' for idx, v in ipairs(items) do if idx % 2 == 1 then -- it is likely possible to issue a HMGET resultkey = redis.call('HGET', ii, v) else result[resultkey] = tonumber(v) end end return cjson.encode(result)) get-index load-dataz fetch-featurez fetch-sample)r r Tc Csddl}ddl}ddl}|}|j}|j|}|j|}tj D]v\}}|rd|tj krdqL| | d} d|| f} |j|dd|d|dd | |dd d |} | | ksLtqLdS) zLoad scripts into Redis Parameters ---------- read_only : bool, optional If True, only load read-only scripts. If False, load writable scripts rNasciiz scripts/%s/%shostnamez /SCRIPT/LOAD)datastateHSETHGET scripts/%s)redbiomredbiom._requestshashlib get_config _requests get_session make_postmake_getr _scriptsitems_admin_scriptssha1encode hexdigestputAssertionError) Z read_onlyrrconfigrpostgetnamescriptrZkeypairobsrrr load_scriptsss      zScriptManager.load_scriptscCsf|tjkrtj|Sddl}ddl}|}|j|}|ddd|}|dkrXtd|tj|<|S)zRetreive the SHA1 of a script Parameters ---------- name : str The name of the script to fetch Raises ------ ValueError If the script name is not recognized rNrrrzUnknown script)r _cacherrrrr ValueError)r'rr$r&sharrr r&s    zScriptManager.getcCsPddl}ddl}|}|j}||dd||ddit_dS)z.Flush the loaded scripts in the redis databaserNrz /SCRIPT/FLUSHz/DEL/state:scripts)rrrrrr&r r+)rr$rrrr drop_scriptss zScriptManager.drop_scriptsN)T) __name__ __module__ __qualname____doc__rrr+ staticmethodr*r&r.rrrr r s c # r cCsPddl}ddl}ddl}|}|j|}|jd}|ddd|dS)aCreate a new timestamp in the database Notes ----- Time is represented as "%d.%b.%Y" (e.g., 25.Jul.2019). Timestamps are pushed into an array such that index 0 is the latest timestamp. A reasonable interpretation of this field, and the use of this method, is to obtain the timestamps of when the database was last updated. Redis command summary --------------------- LPUSH state:timestamps rNz%d.%b.%YrZLPUSHz timestamps/%s)rrdatetimerrrnowstrftime)rr4r$r%fmtrrr create_timestamps r8cCs0ddl}ddl}|}|j|}|dddS)zwObtain the stored timestamps Redis command summary --------------------- LRANGE state:timestamps 0 -1 rNrZLRANGEztimestamps/0/-1)rrrrr)rr$r&rrr get_timestampss  r9cCsddl}ddl}|}|j|}z*|ddd||f||dd|jWn(ddl}td||jdYnXt dS) aCreate a context within the cache Parameters ---------- name : str The name of the context, e.g., deblur@150nt description : str A brief description about the context, e.g., "Default quality filtering, followed by application of Deblur with a trim length of 150nt." Redis commmand summary ---------------------- HSET state:context HSET :state db-version rNrrzcontexts/%s/%szstate/db-version/%szUnable to create context: %s)file) rrrrrZ__db_version__sysprintstderrr r*)r' descriptionrr$r%r;rrr create_contexts r?c s|ddl}ddl}|dkrd}n|dkr,d}n td||} |j| } td} tdt t ||} t || D]} g}g}| D]}|j ||d d }t |j t |j dd std |t}fd d||jD}dddt||j D}||d|d|||qtt |}| |g||}| ddd||dd| }| |d|d dqxdS)aManage the loading of data for a particular axis Parameters ---------- table : biom.Table The table to obtain data from ids : iterable of str The IDs to obtain data for opposite_ids : iterable of str The IDs of the opposite axis in the table opposite_id_index : dict The index which maps an opposite ID to the index value within the Redis database for the identifier axis_label : str The biom.Table axis label of ids context : str The context to load the data into batchsize : int The number of identifiers to group into a single request Notes ----- This method only supports count data. Data are loaded through the "load-data" Lua script managed in the ScriptsManager. This method in effect packs the data into a structure compatible with Webdis, and the EVALSHA command structure of Redis. The "load-data" script then iterates over the "KEYS" and "ARGV"s, parsing the respective entries into values that can be directly loaded. Redis command summary --------------------- EVALSHA N :: ... ... Note that "N" refers to the number of "KEYS". The "load-data" Lua script assumes that there are "N" "KEYS" as well as "N" "ARGV"s. For the call, "KEYS" are the prefixed identifiers (e.g., "::") and "ARGV" are the "packeddata". "KEYS" and "ARGV" are expected to be in index order with each other. rNfeature observationsamplez%s is unrecognized as an axisr F)axisZdensegzData do not appear to be countscsg|] }|qSrr.0iopposite_id_indexrr Lsz#_load_axis_data..|cSsg|]\}}d||fqS)z%d|%dr)rFrGvrrr rJOs:EVALSHA/zs-represented/%sSADD)verbose)rrr,rrrr r&maxrlennp array_splitrZallcloseroundZastypeintindicesjoinzipappendstr)tableidsZ opposite_idsrIZ axis_labelcontext batchsizerrDr$r%Z loader_shasplitsbatchkeysargvZid_valuesZ int_valuesZremappedpackednkeyspayloadrrHr _load_axis_datas@*         riFc s@ddl}ddl}ddl}|}|j|}|j|}|j||t||||}| dd} |j dddd} ddt | t || dD} ddt | t || d D} t || | | d |d d t || | | d|d d t |j dd|jdd} | dk r8||d d|jj}dd| D}||dd||ddd}|D]$}t |D]\}}|||_qLq@| jddD]ĉsrg}g}jD]4}|r|d||jn ||jqd|}||ddj|ffddjD}||ddd||rrd|}||ddj|fqrt| S)aLoad nonzero sample data. Parameters ---------- table : biom.Table The BIOM table to load. context : str The context to load into. tag : str A tag to associated the samples with (e.g., a preparation ID). redis_protocol : bool, optional Generate commands for bulk load instead of HTTP requests. batchsize : int, optional The number of samples or features to load at once Raises ------ ValueError If the context to load into does not exist. If a samples metadata has not already been loaded. If a table is empty. Notes ----- This method does not support non count data. The feature IDs are remapped into an integer space to reduce memory consumption as sOTUs are large. The index is maintained in Redis under :feature-index and :feature-index-inverted. The data are stored per sample with keys of the form "data:". The string stored is tab delimited, where the even indices (i.e .0, 2, 4, etc) correspond to the unique index value for an feature ID, and the odd indices correspond to the counts associated with the sample/feature combination. Redis command summary --------------------- EVALSHA 1 :feature-index EVALSHA 1 :sample-index LPUSH :samples: ... LPUSH :features: ... SADD :samples-represented ... SADD :features-represented ... Returns ------- int The number of samples loaded. rNrArDcSsi|]\}}||qSrrrFrGjrrr sz$load_sample_data..r@cSsi|]\}}||qSrrrlrrr rnsrB )r`irzstate/has-taxonomy/1cSsi|] }|j|qSrr')rFnrrr rnsZHMGETdz feature-index)r& buffer_sizeZmultikeyF)Z include_selfz has-terminalrOrPztaxonomy-children:%s/%scsg|]}d|jjfqSz%s/%srprFcnoderr rJsz$load_sample_data..HMSETztaxonomy-parents/%szterminal-of:%s/%s)rr redbiom.utilrrrrvalid_stage_for_loadr^rZ get_indexri_metadata_to_taxonomy_treemetadatabufferedZtipsr' postorderis_tipchildrenr[rYrS)r]r_tagZredis_protocolr`rr$r&r%samplesr)Z obs_indexZ samp_indextaxonomyZhmgetterZ tip_namesZids_ZblkentityidxpackZ terminal_packrvrfZid_packrrwr load_sample_data`st4                 rcCs|dkr dSddl}|jddt||D}t|D]N}|rJq<|rTq.__)skbioZTreeNodeZ from_taxonomyrZlistrris_rootrSr'endswithparentextendrremove)r^rrtrxrrrr r~s    r~csddl}ddl}ddl}ddl}|}|j|}|j|}|j|}|j j | }|j ddkrr| }dk r||j ddd}|j |stdfdd||j dD||j d<|j|j ddd|d d d } |jtt|jt| }t|dkrdS|j} |j } |D]B\} } fd dt|j | jD}d | }|d d|||q,| D]B}fddt|j||D}d|d|f}|d d|qtdd|j}|d d|dd|j }|d d|t| S)a@Load sample metadata. Parameters ---------- md : pd.DataFrame QIIME or Qiita compatible metadata. tag : str, optional A tag associated with the information being loaded such as a preparation ID. Notes ----- Values considered to be non-informative are omitted from load. TODO: expose a stable list of the nullables, see #19 Returns ------- int The number of samples loaded. Redis command summary --------------------- SMEMBERS metadata:samples-represented SET metadata:categories: HMSET metadata:category: ... SADD metadata:samples-represented ... ... SADD metadata:categories-represented ... rNz #SampleIDZ sample_name%Sample metadata must be loaded first.csg|]}d|fqSz%s_%srrErrr rJ8sz(load_sample_metadata..TZinplacerSMEMBERSsamples-representedcsg|]\}}t|r|qSr) _indexable)rFrvrG null_valuesrr rJFs z categories:%sZSETcs.g|]&\}}t|rd|tt|fqSrt)rrr\)rFrrLrrr rJNs zcategory:%s/%srOryzsamples-represented/%srPzcategories-represented/%s)jsonrrrzrrrZmake_putrutilZ NULL_VALUEScopycolumns reset_indexhas_sample_metadatar, set_indexlocrsetindexrSZiterrowsrZredumpsrY)mdrrrr$r%r"r&Z original_ids representedrZindexed_columnsrrowrkeycolZbulk_setrhr)rrr load_sample_metadatasN    &    rcs\ddl}ddl}ddl}ddl}|}|j|}|}|jddkrR| }dk rfdd||jdD||jd<|j |jddd|j t |jstd|j |}|D]&\}}d |d |f} |d d | qt|} d d|jD} |j |j| |jd}|D](\}} d|d | f} |d d | q"t|} | | fS)aLoad stem -> sample associations Parameters ---------- md : pd.DataFrame QIIME or Qiita compatible metadata. tag : str, optional A tag associated with the information being loaded such as a preparation ID. Notes ----- Values considered to be non-informative are omitted from load. Returns ------- int The number of stems based on metadata values found. int The number of stems based on the categories found. Redis command summary --------------------- SADD metadata:text-search: ... SADD metadata:category-search: ... rNrcsg|]}d|fqSrrrErrr rJsz4load_sample_metadata_full_search..Trrztext-search:%s/%srOrrPcSsg|]}|ddqS)_ )rrurrr rJs)rzcategory-search:%s/%s)rrrzZpandasrrrrrrrrrrrr,Z df_to_stemsrrYrSZ DataFrame)rrrpdr$r%ZstemsstemrrhZ value_stems categoriesZcatsZ cat_stemsrrr load_sample_metadata_full_search]s8 &  rcCs||kS)z?Returns true if the value appears to be something that storabler)valueZ nullablesrrr rsrc@s eZdZdS) AlreadyLoadedN)r/r0r1rrrr rsrcsddl}dkrd|jfdd|Ddd}t|}|sLtd||d d }t|}||}|sx|rxtd |j|std |||jd dddS)aTag samples, reduce to only those relevant to load Parameters ---------- table : biom.Table The table to operate on context : str The context to load into get : make_get instance A getter tag : str, optional The tag to apply to the samples Raises ------ ValueError If a samples metadata has not already been loaded. ValueError If the table is empty. AlreadyLoaded If the table appears to already be loaded. Returns ------- biom.Table A copy of the input table, filtered to only those samples which are novel to the context. Sample IDs reflect tag. rNZUNTAGGEDcsi|]}|d|fqSrrrErrr rnsz#_stage_for_load..FrzThe table is empty.rrz'The table appears to already be loaded.rcSs |dkS)Nr)sum)rLrGrrrr z!_stage_for_load..rArk) rzZ update_idsr^rr,rrrfilter)r]r_r&rrrrZto_loadrrr r|s$    r|rrc Csddl}ddl}ddl}|}|j|}td}d||f} g} tdt t ||} t || D]<} d} || | gt | }||ddd|}| |qh| S) aGet a unique integer value for a key within a context Parameters ---------- context : str The context to operate in keys : list or tuple of str The keys to get a unique index for axis : str Either feature or sample batchsize : int, optional The number of IDs to query at once Notes ----- This method is an atomic equivalent of: def get_or_set(d, item): if item not in d: d[item] = len(d) return d[item] Returns ------- tuple of int The unique integer indices within the context for the keys. This is returned in index order with keys. rNr z %s:%s-indexrC1rNrO)rrrrrrr r&rRrrSrTrUrloadsrYr)r_rcrDr`rrr$r%Z indexer_shaZ context_axisrXrarbrgrhrrrr r}s    r})NFrj)N)N)N)rr) urllib.parserrmathrnumpyrTr r8r9r?rirr~rrrr,rr|r}rrrr s&  8 X x) \ C 7