U _W@sZddlZddlZddlZddlZddlZddlZddlZddlZddlm Z ddl m Z ddl m Z ddl m Z ddl mZddl mZddl mZdd l mZddlZed ZeeZeZeejeeeed d ZGd ddeZ Gddde Z!Gddde Z"dddddddddddddej#de j$ddddddfddZ%dS)N)dedent) constants)version)bins)helpers)feature) interface) iteratorsz)%(asctime)s - %(levelname)s - %(message)scCst|dkrtd|dS)zH As things change from version to version, deal with them here. rzunhandled kwarg in %sN) ValueErrorlen TypeError)kwargsr.lib/python3.8/site-packages/gffutils/create.pydeprecation_handlers rc@seZdZddddddddddddddejejddfddZd-d d Zd d Z d dZ ddZ d.ddZ ddZ ddZddZddZddZddZdd Zd!d"Zd#d$Zd%d&Zd'd(Zd)d*Zd+d,ZdS)/ _DBCreatorFNerror zutf-8TcKs~||_|dkrg}|dkr^tddg|r4tdt|ddg}|D]}td|qJ||_||_||_| |_ |dkrg}||_ |std d } d }| |_ ||_ |rt j|rt |||_||_t|tjrt|}n|}||_tj|j_|||dk r2|jd kr*td |||j_||_ tj!|_"t#j$|||| | | d |_%d|krn|d|_&n t'(t)|_&dS)zk Base class for _GFFDBCreator and _GTFDBCreator; see create_db() function for docs Nmergestartendz8Can't merge start/end fields since they must be integersframestrandz\%s field will be merged for features with the same ID; this may result in unusable features.z'infer_gene_extent' will be deprecated. For now, the following equivalent values were automatically set: 'disable_infer_genes=True', 'disable_infer_transcripts=True'. Please use these instead in the future.Tdebugzsetting text factory to %s)data checklines transformforce_dialect_check from_stringdialect_autoincrements)*_keep_tempfilesset intersectionr warningswarnforce_merge_fieldspragmasmerge_strategydefault_encoding directivesdisable_infer_genesdisable_infer_transcriptsospathexistsunlinkdbfnid_spec isinstancesix string_typessqlite3ZconnectconnZRowZ row_factory set_verboseverboseloggerr text_factory_datalevelZ_orig_logger_levelr DataIteratoriteratorr collections defaultdictint)selfrr1forcer9r2r(rrrrrr)r+r,infer_gene_extentr&r;r'r!r*r r%wr7rrr__init__2sp           z_DBCreator.__init__cCs>|dkrttjn|r(ttjn ttj||_dS)Nr)r:setLevelloggingDEBUGINFOZERRORr9)rCr9rrrr8s  z_DBCreator.set_verbosecCs$|j|d7<d||j|fS)Nz%s_%s)r )rCkeyrrr_increment_featuretype_autoidsz(_DBCreator._increment_featuretype_autoidc CsLt|jtjr|jg}nnt|jdr.|jg}nXt|jtrz"|j|j}t|tjrZ|g}Wqtk r|||jYSXn|j}|D]}t|dr||}|r| dr||ddS|Sqt |dkr|ddkr|ddkrt ||d dSz|j |dWStt fk r<YqXq||jS) z Given a Feature from self.iterator, figure out what the ID should be. This uses `self.id_spec` identify the ID. __call__zautoincrement:Nr:rL)r3r2r4r5hasattrdict featuretypeKeyErrorrN startswithr getattr attributes IndexError)rCfZid_keykZ_idrrr _id_handlers4          *z_DBCreator._id_handlercCs:|j}|tjd|f}tjfd|jj i|S)Nz WHERE id = ?r) r7cursorexecuter_SELECTfetchonerFeaturer?r)rCIDcresultsrrr _get_features  z_DBCreator._get_featurec s|dkrtd|dkr6tdd|fS|dkrF|fS|dkrg}|jdkr|td d d |Dttt j dd  |j }|D]z}d }|D] }t ||t |krd}qq|r|||jdkrtd|fq|jdkrtd|fqt|dkr\j} |jdd\} }|| | j| |fS|jdkrztdt|tj} tfdd |j D} |D]|}|jdkrtd|f|jD](}| |g} | ||| | |<q|j D]}| |t ||gqq| D]\}} tt| | |<q(| |_| D]&\}} t||dttt | qR|jdkrtd|||fSn,|dkr|!j_|fStd|dS)a? Different merge strategies upon name conflicts. "error": Raise error "warning" Log a warning, which indicates that all future instances of the same ID will be ignored "merge": Combine old and new attributes -- but only if everything else matches; otherwise error. This can be slow, but is thorough. "create_unique": Autoincrement based on the ID, always creating a new ID. "replace": Replaces existing database feature with `f`. rzDuplicate ID {0.id}warningzCDuplicate lines in file for id '{0.id}'; ignoring all but the firstNreplacerrzcandidates with same idspec: %scSsg|] }|jqSr)id.0irrr sz(_DBCreator._do_merge..rSTFz2same attributes between: existing: %s this : %sz7different attributes between: existing: %s this : %sr create_unique)r(znum candidates: %scs g|]}|tt|gfqSr)r"rYrlfieldr\rrrn6sz merging %s %s ,z MERGED: %szInvalid merge strategy '%s')"r formatr:rhr9r_candidate_mergeslistr"rZ_gffkeys differencer&rYappendr rj _do_merge_add_duplicatecopydeepcopyrZrUkeys setdefaultextendupdateitemssetattrjoinsortedmapstrrN)rCr\r(Z add_duplicateZfeatures_to_mergeZ_gffkeys_to_checkZexisting_featureZother_attributes_samer]Zorig_idZuniqued_featureZmerged_attributesZ final_fieldsvrqrrrrrys                   z_DBCreator._do_mergec Cs|j}z|d||fWn6tjk rT|d||j||jfYnX|jdkrrt d||f|j dS)aR Adds a duplicate ID (as identified by id_spec) and its new ID to the duplicates table so that they can be later searched for merging. Parameters ---------- newid : str The primary key used in the features table idspecid : str The ID identified by id_spec zg INSERT INTO duplicates (idspecid, newid) VALUES (?, ?)rzadded id=%s; new=%sN) r7r_r`r6ProgrammingErrordecoder)r9r:rcommit)rCZidspecidZnewidrerrrrz\s     z_DBCreator._add_duplicatecCsb||jg}|j}|tjd|jf}|D]"}|tj fd|j j i|q2t t |S)z Identifies those features that originally had the same ID as `f` (according to the id_spec), but were modified because of duplicate IDs. zh JOIN duplicates ON duplicates.newid = features.id WHERE duplicates.idspecid = ?r)rgrjr7r_r`rrarxrrcr?rrvr")rCr\Z candidatesrerfrmrrrru~s z_DBCreator._candidate_mergescCstdSNNotImplementedError)rClinesrrr_populate_from_linessz_DBCreator._populate_from_linescCstdSrrrCrrr_update_relationssz_DBCreator._update_relationscCs2|j}tjD]}|d|fq|jdS)NzDROP INDEX IF EXISTS ?)r7r_rZINDEXESr`r)rCreindexrrr _drop_indexess  z_DBCreator._drop_indexescCs>||_|j}|ddd|jD|jdS)a Set pragmas for the current database connection. Parameters ---------- pragmas : dict Dictionary of pragmas; see constants.default_pragmas for a template and http://www.sqlite.org/pragma.html for a full list. z; cSsg|] }d|qS)z PRAGMA %s=%srrkrrrrnsz*_DBCreator.set_pragmas..N)r'r7r_ executescriptrrr)rCr'rerrr set_pragmass  z_DBCreator.set_pragmascCs6|j}tj}||j|tj|j dS)z Table creation N) r7r_r6Zsqlite_version_inforr'rrZSCHEMAr)rCrerrrr _init_tabless    z_DBCreator._init_tablescCs&|j}|j|jj}|ddd|D|dttjt |jj d|dt |j td|d|d td |d |d td |d|dtd|d|dtd|d|dtd|d|j|jj|_dS)z Various last-minute stuff to perform after file has been parsed and imported. In general, if you'll be adding stuff to the meta table, do it here. zO INSERT INTO directives VALUES (?) css|] }|fVqdSrrrkrrr sz'_DBCreator._finalize..zX INSERT INTO meta (version, dialect) VALUES (:version, :dialect))rrzM INSERT OR REPLACE INTO autoincrements VALUES (?, ?) Creating relations(parent) index$DROP INDEX IF EXISTS relationsparent2CREATE INDEX relationsparent ON relations (parent)Creating relations(child) index#DROP INDEX IF EXISTS relationschild0CREATE INDEX relationschild ON relations (child)z$Creating features(featuretype) indexz DROP INDEX IF EXISTS featuretypez2CREATE INDEX featuretype ON features (featuretype)z+Creating features (seqid, start, end) indexz"DROP INDEX IF EXISTS seqidstartendz:CREATE INDEX seqidstartend ON features (seqid, start, end)z3Creating features (seqid, start, end, strand) indexz(DROP INDEX IF EXISTS seqidstartendstrandzHCREATE INDEX seqidstartendstrand ON features (seqid, start, end, strand)zRunning ANALYZE featureszANALYZE featuresN)r7r_r*r? executemanyr`rUrr_jsonifyrrvr rr:inforr$)rCrer*rrr _finalizesD                      z_DBCreator._finalizecCs(|||j||dS)zb Calls various methods sequentially in order to fully build the database. N)rrr?rrrrrrcreates z_DBCreator.createcCs|||dSr)rr)rCr?rrrrs z_DBCreator.updateccs(|j}||}|D] }|VqdS)z; Execute a query directly on the database. N)r7r_r`)rCZqueryreresultrmrrrr`s  z_DBCreator.executec CsHz|tj|Wn,tjk rB|tj||jYnXdSz5 Insert a feature into the database. N)r`r_INSERTastupler6rr)rCrr_rrr_insert s z_DBCreator._insertc Cs`z"|tjt||jgWn8tjk rZ|tjt||j |jgYnXdSr) r`rZ_UPDATErvrrjr6rrr)rrrr_replacesz_DBCreator._replace)N)F)__name__ __module__ __qualname__r6OptimizedUnicoderdefault_pragmasrGr8rNr^rgryrzrurrrrrrrrr`rrrrrrr1sH Q 6 " 9   rcs,eZdZfddZddZddZZS) _GFFDBCreatorcstt|j||dS)z _DBCreator subclass specifically for working with GFF files. create_db() delegates to this class -- see that function for docs N)superrrGrCargsr  __class__rrrG&sz_GFFDBCreator.__init__c s|j}|d}tdd}d}gg}}t|D]P\}} |}|jrt|ddkrttj ||tj | | | _ z| | |Wntjk r\|| |j\} | dkr*|dtjj f|jrXddd |jD} fd d |jDj g} |d | t| n.| d krB|| |n| d krX| | |YnXd| jkr:| jdD]} |d| | j fqrq:|dkrtd|j|jrt||dS)NrzPopulating featuresAPopulating features table and first-order relations: %d features ry UPDATE features SET attributes = ? WHERE id = ? , cSsg|] }d|qSz%s = ?rrprrrrn\sz6_GFFDBCreator._populate_from_lines..csg|]}t|qSrrYrpfixedrrrn^sy UPDATE features SET %s WHERE id = ? riroZParentzz INSERT OR IGNORE INTO relations VALUES (?, ?, 1) .No lines parsed -- was an empty file provided?)r7r_rr:r enumerater9sysstderrwriteflushr^rjrr6IntegrityErrorryr(r`rrrZr&rtuplerr r)rCrre last_percmsgZ features_seenZ _featuresZ _relationsrmr\final_strategy _set_clausevaluesparentrrrr.sr                z"_GFFDBCreator._populate_from_linesc std|j}|j}|j}t|jtjr>|j}nd}tj d|dj }t |dT| d|D]>}| dt ||D]$}d|d |d fd qqlW5QRXfd d }|d || d| d|j|jstj dS)NzUpdating relations .gffutilsFdeletesuffixrFzSELECT id FROM featuresz SELECT child FROM relations WHERE parent IN (SELECT child FROM relations WHERE parent = ?)  r c 3sFtj2}|D]&}|d\}}t||ddVqW5QRXdS)Nr)rchildr=)opennamestripsplitrU)finlinerrfoutrrrelations_generators z<_GFFDBCreator._update_relations..relations_generatorzf INSERT OR IGNORE INTO relations VALUES (:parent, :child, :level) zDROP INDEX IF EXISTS binindexz'CREATE INDEX binindex ON features (bin))r:rr7r_r3r!r4r5tempfileNamedTemporaryFilerrr`rrrrrr-r0) rCrec2Zc3rtmprZ grandchildrrrrr{s4     .    z_GFFDBCreator._update_relationsrrrrGrr __classcell__rrrrr%s Mrcs,eZdZfddZddZddZZS) _GTFDBCreatorcs@|dd|_|dd|_|dd|_tt|j||dS)zS create_db() delegates to this class -- see that function for docs transcript_key transcript_idgene_keygene_id subfeatureexonN)poprrrrrrGrrrrrGsz_GTFDBCreator.__init__c sjd}|j}d}d}d}t|D]\}}||krj|jdkrP|jsPtdn|jdkrj|jsjtd|d}|jr|ddkrt j ||t j | ||_z|||Wntjk r|||j\} | d krP|d tjjf|jr~d d d |jD} fdd |jDjg} |d| | n.| dkrh|||n| dkr~|||YnXg} d} d}|j|jkr|j|jd} | | |jdf|j|jkr|j|j}t|dkr|d}| ||jdf| dk r| || df|d| q"|dkr.csg|]}t|qSrrrprrrrnsrrirorzy INSERT OR IGNORE INTO relations (parent, child, level) VALUES (?, ?, ?) rCommitting changes)$r7r_rrVr,r$r%r+r9rrrrr^rjrr6rryr(r`rrrZr&rrrrxrr rr r:rr)rCrrreZgene_and_transcript_check_limitrZ lines_seenrmr\rrrZ relationsrZ grandparentrrrrs              z"_GTFDBCreator._populate_from_linescsjrjrdSj}j}td|d|dtd|d|djsrjsrd}njr~d}n jrd }td |tjt j rj}nd }t j d |d j }t|dD|_|djfd}d}|D]\}} js|d|jf|\} } } } j|gj| gi}tj| | dd}dtt|| | | | d |t|gd|d7}js| |kr|d| jf|\}}} } j| gi}tj||dd}dtt| | ||| d|t|gd| }|d7}qW5QRXfdd}|d|dtdd}t|D]\}}t|t|d}||krtjd|||ftj |}z!||WnBt"j#k r$|d\}}|dt|j%|j&fYnXqPtdj'jst()j dS) Nrrrrrrzgene and transcriptrrz,Inferring %s extents and writing to tempfilerFrrFa SELECT DISTINCT firstlevel.parent, relations.parent FROM ( SELECT DISTINCT parent FROM relations JOIN features ON features.id = relations.child WHERE features.featuretype = ? AND relations.level = 1 ) AS firstlevel JOIN relations ON firstlevel.parent = child WHERE relations.level = 1 ORDER BY relations.parent ra  SELECT MIN(start), MAX(end), strand, seqid FROM features JOIN relations ON features.id = relations.child WHERE parent = ? AND featuretype == ? T)ZonerrrLa8 SELECT MIN(start), MAX(end), strand, seqid FROM features JOIN relations ON features.id = relations.child WHERE parent = ? AND featuretype == ? c 3sddddddddg}tj}|D]z}ttt||d }|dd |d <d |d <d |d<g|d<t |d|d<t j f|} ||_ |Vq$W5QRXdS)zS Generator of items from the file that was just created... rseqidrrrrVbinrZr.ZscoreZgffutils_derivedsourcerZextraN)rrrUrvziprrrrZ _unjsonifyrrcr^rj)r}rrdr\rrCrrderived_feature_generators"     zB_GTFDBCreator._update_relations..derived_feature_generatorz#Importing inferred features into dbdz%s of %s (%s%%) rzm UPDATE features SET attributes = ? WHERE id = ? r)*r+r,r7r_r:rr`r3r!r4r5rrrrZ_tmpfilerrbrrrrrrrrrrrBfloatrrrrr6rryrZrjrr-r0)rCrerrrrZ last_gene_idZ n_featuresrrZtranscript_startZtranscript_endrrZtranscript_attributesZtranscript_binZ gene_startZgene_endZgene_attributesZgene_binrrrmr\percrrrrrr0s                           z_GTFDBCreator._update_relationsrrrrrrs rrFrrrrrTc stt|tfddtjD}tjf|}|jf|dkrL|j}|j |d<|j |d<d|d<| sx|dd krt }|pd }t|d }n.|dd krt }|pd dd}t|| | |d}|jf|||d<|f|}| |dkrtj|j||||d}ntj|||||d}|S)a$ Create a database from a GFF or GTF file. For more details on when and how to use the kwargs below, see the examples in the online documentation (:ref:`examples`). Parameters ---------- data : string or iterable If a string (and `from_string` is False), then `data` is the path to the original GFF or GTF file. If a string and `from_string` is True, then assume `data` is the actual data to use. Otherwise, it's an iterable of Feature objects. dbfn : string Path to the database that will be created. Can be the special string ":memory:" to create an in-memory database. id_spec : string, list, dict, callable, or None This parameter guides what will be used as the primary key for the database, which in turn determines how you will access individual features by name from the database. If `id_spec=None`, then auto-increment primary keys based on the feature type (e.g., "gene_1", "gene_2"). This is also the fallback behavior for the other values below. If `id_spec` is a string, then look for this key in the attributes. If it exists, then use its value as the primary key, otherwise autoincrement based on the feature type. For many GFF3 files, "ID" usually works well. If `id_spec` is a list or tuple of keys, then check for each one in order, using the first one found. For GFF3, this might be ["ID", "Name"], which would use the ID if it exists, otherwise the Name, otherwise autoincrement based on the feature type. If `id_spec` is a dictionary, then it is a mapping of feature types to what should be used as the ID. For example, for GTF files, `{'gene': 'gene_id', 'transcript': 'transcript_id'}` may be useful. The values of this dictionary can also be a list, e.g., `{'gene': ['gene_id', 'geneID']}` If `id_spec` is a callable object, then it accepts a dictionary from the iterator and returns one of the following: * None (in which case the feature type will be auto-incremented) * string (which will be used as the primary key) * special string starting with "autoincrement:X", where "X" is a string that will be used for auto-incrementing. For example, if "autoincrement:chr10", then the first feature will be "chr10_1", the second "chr10_2", and so on. force : bool If `False` (default), then raise an exception if `dbfn` already exists. Use `force=True` to overwrite any existing databases. verbose : bool Report percent complete and other feedback on how the db creation is progressing. In order to report percent complete, the entire file needs to be read once to see how many items there are; for large files you may want to use `verbose=False` to avoid this. checklines : int Number of lines to check the dialect. merge_strategy : str One of {merge, create_unique, error, warning, replace}. This parameter specifies the behavior when two items have an identical primary key. Using `merge_strategy="merge"`, then there will be a single entry in the database, but the attributes of all features with the same primary key will be merged. Using `merge_strategy="create_unique"`, then the first entry will use the original primary key, but the second entry will have a unique, autoincremented primary key assigned to it Using `merge_strategy="error"`, a :class:`gffutils.DuplicateID` exception will be raised. This means you will have to edit the file yourself to fix the duplicated IDs. Using `merge_strategy="warning"`, a warning will be printed to the logger, and the duplicate feature will be skipped. Using `merge_strategy="replace"` will replace the entire existing feature with the new feature. transform : callable Function (or other callable object) that accepts a `Feature` object and returns a (possibly modified) `Feature` object. gtf_transcript_key, gtf_gene_key : string Which attribute to use as the transcript ID and gene ID respectively for GTF files. Default is `transcript_id` and `gene_id` according to the GTF spec. gtf_subfeature : string Feature type to use as a "gene component" when inferring gene and transcript extents for GTF files. Default is `exon` according to the GTF spec. force_gff : bool If True, do not do automatic format detection -- only use GFF. force_dialect_check : bool If True, the dialect will be checkef for every feature (instead of just `checklines` features). This can be slow, but may be necessary for inconsistently-formatted input files. from_string : bool If True, then treat `data` as actual data (rather than the path to a file). keep_order : bool If True, all features returned from this instance will have the order of their attributes maintained. This can be turned on or off database-wide by setting the `keep_order` attribute or with this kwarg, or on a feature-by-feature basis by setting the `keep_order` attribute of an individual feature. Note that a single order of attributes will be used for all features. Specifically, the order will be determined by the order of attribute keys in the first `checklines` of the input data. See helpers._choose_dialect for more information on this. Default is False, since this includes a sorting step that can get time-consuming for many features. infer_gene_extent : bool DEPRECATED in version 0.8.4. See `disable_infer_transcripts` and `disable_infer_genes` for more granular control. disable_infer_transcripts, disable_infer_genes : bool Only used for GTF files. By default -- and according to the GTF spec -- we assume that there are no transcript or gene features in the file. gffutils then infers the extent of each transcript based on its constituent exons and infers the extent of each gene bases on its constituent transcripts. This default behavior is problematic if the input file already contains transcript or gene features (like recent GENCODE GTF files for human), since 1) the work to infer extents is unnecessary, and 2) trying to insert an inferred feature back into the database triggers gffutils' feature-merging routines, which can get time consuming. The solution is to use `disable_infer_transcripts=True` if your GTF already has transcripts in it, and/or `disable_infer_genes=True` if it already has genes in it. This can result in dramatic (100x) speedup. Prior to version 0.8.4, setting `infer_gene_extents=False` would disable both transcript and gene inference simultaneously. As of version 0.8.4, these argument allow more granular control. force_merge_fields : list If merge_strategy="merge", then features will only be merged if their non-attribute values are identical (same chrom, source, start, stop, score, strand, phase). Using `force_merge_fields`, you can override this behavior to allow merges even when fields are different. This list can contain one or more of ['seqid', 'source', 'featuretype', 'score', 'strand', 'frame']. The resulting merged fields will be strings of comma-separated values. Note that 'start' and 'end' are not available, since these fields need to be integers. text_factory : callable Text factory to use for the sqlite3 database. See https://docs.python.org/2/library/ sqlite3.html#sqlite3.Connection.text_factory for details. The default sqlite3.OptimizedUnicode will return Unicode objects only for non-ASCII data, and bytestrings otherwise. pragmas : dict Dictionary of pragmas used when creating the sqlite3 database. See http://www.sqlite.org/pragma.html for a list of available pragmas. The defaults are stored in constants.default_pragmas, which can be used as a template for supplying a custom dictionary. sort_attribute_values : bool All features returned from the database will have their attribute values sorted. Typically this is only useful for testing, since this can get time-consuming for large numbers of features. _keep_tempfiles : bool or string False by default to clean up intermediate tempfiles created during GTF import. If True, then keep these tempfile for testing or debugging. If string, then keep the tempfile for testing, but also use the string as the suffix fo the tempfile. This can be useful for testing in parallel environments. Returns ------- New :class:`FeatureDB` object. c3s|]}||fVqdSrrrkZ_localsrrrszcreate_db..Nrr*rrZfmtZgff3rd)r2Zgtfrr)rr)rrrr2rz:memory:) keep_orderr'sort_attribute_valuesr;)localsrrUrZ_iterator_kwargsr r>rrZ_iterr*rrrrZ FeatureDBr7)rr1r2rDr9rr(rZgtf_transcript_keyZ gtf_gene_keyZgtf_subfeatureZ force_gffrrrr;r&r'rrr!rEr+r,r r?clsZ add_kwargsreZdbrrr create_dbsX]        r)&r{r$r@rrr-r6r4textwraprZgffutilsrrrrrrr rIZ Formatter formatterZ getLoggerrr:Z StreamHandlerZchrHrJZ setFormatterZ addHandlerrobjectrrrrrrrrrrsj             wI