+gdddlZddlZddlZddlmZmZddlmZddlm Z dZ dZ gdZ gdZ d efd ZGd d eZdS) N)ArgumentParser Namespace)BaseDatasetsCLICommand) get_loggerz><<<<<<< This should probably be modified because it mentions: z======= >>>>>>> )TextEncoderConfigByteTextEncoderSubwordTextEncoderencoder_configmaybe_build_from_corpus manual_dir) )z tfds\.coredatasets)ztf\.io\.gfile\.GFileopen)z tf\.([\w\d]+)zdatasets.Value('\1'))ztfds\.features\.Text\(\)zdatasets.Value('string'))ztfds\.features\.Text\(zdatasets.Value('string'),)z+features\s*=\s*tfds.features.FeaturesDict\(zfeatures=datasets.Features()ztfds\.features\.FeaturesDict\(zdict()zThe TensorFlow Datasets AuthorszDThe TensorFlow Datasets Authors and the HuggingFace Datasets Authors)ztfds\.z datasets.)zdl_manager\.manual_dirzself.config.data_dir)zself\.builder_configz self.configargsc6t|j|jS)zz Factory function used to convert a model TF 1.0 checkpoint in a PyTorch checkpoint. Returns: ConvertCommand )ConvertCommand tfds_pathdatasets_directory)rs 9lib/python3.11/site-packages/datasets/commands/convert.pyconvert_command_factoryr*s $.$*A B BBc@eZdZedefdZdedefdZdZdS)rparserc|dd}|dtdd|dtdd |t d S) z Register this command to argparse so it's available for the datasets-cli Args: parser: Root parser to register command-specific arguments convertzHConvert a TensorFlow Datasets dataset to a HuggingFace Datasets dataset.)helpz --tfds_pathTzQPath to a TensorFlow Datasets folder to convert or a single tfds file to convert.)typerequiredrz--datasets_directoryz(Path to the HuggingFace Datasets folder.)funcN) add_parser add_argumentstr set_defaultsr)r train_parsers rregister_subcommandz"ConvertCommand.register_subcommand4s(( [)   !! d "   !! "tBl "    !!'>!?????rrrcJtd|_||_||_dS)Nzdatasets-cli/converting)r_logger _tfds_path_datasets_directory)selfrrrs r__init__zConvertCommand.__init__Ks'!";<< ##5   rc* tj|jr%tj|j}nXtj|jr%tj|j}ntdtj|j}|j d|d|g}g}i}tj|jrtj |}n%tj |jg}|D]}|j d|tj ||}tj ||} tj|r d|vsd|vsd|vr|j dt|d 5} | } dddn #1swxYwYg} d } d }g}| D]}|d vr d vrdvrdvrdndvrd"dvrdndvrddnt#fdt$Drd}t't)fdt$}| t,t/|zdz| | t0t2D]\}}t5j||dvrrt5jd}|d|ddDd |dzd!vsd"vsdvr$td# d$vsd%vrd} | | sd&|vr|dd}tj ||tj |} tj!d'|j d(|"fd)|Dn|| |r|| t| d*d 5} | #| dddn #1swxYwY|j d+| |D]} tj |}||dd}|j d,|d|tIj%||t#tL$r!|j 'd-|d.YwxYw|r#|D]"}|j (d/|d0!dSdS)1NzA--tfds_path is neither a directory nor a file. Please check path.zConverting datasets from z to zLooking at file r*_testz.pyz Skipping filezutf-8)encodingFz!import tensorflow.compat.v2 as tfz @tfds.corez builder=selfz-import tensorflow_datasets.public_api as tfdszimport datasets zimport tensorflowzfrom absl import loggingzfrom datasets import logging getLoggerrc3 K|]}|vV dSN).0 expressionout_lines r z%ConvertCommand.run..s(OOJx/OOOOOOrTc |vSr1r2)er5s rz$ConvertCommand.run..s a8mr tensorflow_datasetsz/from\stensorflow_datasets.*import\s([^\.\r\n]+)c3>K|]}|VdSr1)strip)r3imps rr6z%ConvertCommand.run..s*'Y'Y 'Y'Y'Y'Y'Y'Yr,zfrom . import ztf.ztfds.zError converting GeneratorBasedBuilderBeamBasedBuilderwmt)exist_okzAdding directory ci|]}|Sr2r2)r3r> output_dirs r z&ConvertCommand.run..s.W.W.W3sJ.W.W.Wrwz Converted in zMoving z#Cannot find destination folder for z. Please copy manually.z!You need to manually update file z4 to remove configurations using 'TextEncoderConfig'.))ospathisdirr'abspathisfiledirname ValueErrorr(r&infolistdirbasenamejoinr readlinesreplaceany TO_HIGHLIGHTlistfilterappendHIGHLIGHT_MESSAGE_PREr!HIGHLIGHT_MESSAGE_POST TO_CONVERTresubmatchextendgroupsplitr=makedirsupdate writelinesshutilcopyKeyErrorerrorwarning)r) abs_tfds_pathabs_datasets_path utils_fileswith_manual_updateimports_to_builder_map file_namesf_name input_file output_fileflines out_lines is_builderneeds_manual_update tfds_importsline to_removepattern replacementr`dir_name utils_file dest_folder file_pathr5rFs @@rrunzConvertCommand.runQs 7== ) ) bGOODO<>*-- v1E1ETZIZIZ^ckq^q^q !!/222j7333 &q  & & & & & & & & & & & & & & &IJ"' L+ ++ +7(BB!X--#x//DPP2HH(H44!H/8;;?HH H,,'// \JJHHOOOO,OOOOO J*.' $V,C,C,C,C\%R%R S SI$$%:S^^%Kd%RSSS$$X...$$%;<<<0:JJ,#%6';#I#I)H44H%WYabbE '''Y'Yu{{1~~?S?STW?X?X'Y'Y'YYYY/%++a..@HH$$8(;(;?TX`?`?`$%K9I9I%K%KLLL*h66:LPX:X:X!%J  **** 0Uf__!>>%44W\\*;XFF  gll:v>>  J6666 !!"Bj"B"BCCC&--.W.W.W.W,.W.W.WXXXX"";///" 7"))+666k3999 (Q Y''' ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( L  ;k;; < < < <% n nJ n))*554V^^E25N5NO  !!"IK"I"IZ"I"IJJJ J 4444 n n n ""#l#l#l#lmmmmm n  /    $$w www    s7H&&H* -H* <TT" %T" A0V>>(W)(W)N) __name__ __module__ __qualname__ staticmethodrr$r!r*rr2rrrr3st@N@@@\@,6#636666 rrrrrrr)rIr^rgargparserrdatasets.commandsrdatasets.utils.loggingrr[r\rWr]rrr2rrrs ........444444------]  "C)CCCCPPPPP+PPPPPr