+gd"dZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl m Z ddlmZddlmZddlmZmZmZmZmZmZddlZddlZddlmZddlmZd d l m!Z!m"Z"d d l#m$Z$d d l%m&Z&m'Z'm(Z(m)Z)m*Z*d d l+m,Z,m-Z-m.Z.m/Z/d dl0m1Z1m2Z2d dl3m4Z4m5Z5d dl6m7Z7d dl8m9Z9m:Z:d dl;mZ>m?Z?d dl@mAZAd dlBmCZCmDZDd dlEmFZFd dlGmHZHmIZImJZJd dlKmLZLmMZMmNZNd dlOmPZPd dlQmRZRmSZSd dlTmUZUmVZVmWZWmXZXd dlYmZZZd dl"m[Z[d dl\m]Z]m^Z^d dl_m`Z`d d lambZbmcZcmdZdmeZed d!lfmgZgmhZhmiZimjZjmkZkmlZlmmZmmnZnd d"lompZpmqZqe[jresZtGd#d$euZvGd%d&ewZxGd'd(exZyGd)d*exZze Gd+d,Z{Gd-d.Z|Gd/d0e|Z}Gd1d2e|Z~Gd3d4euZGd5d6e|ZdS)7zDatasetBuilder base class.N) dataclass)partial)Path)DictIterableMappingOptionalTupleUnion)Pool) thread_map)configutils)Dataset)HF_GCP_BASE_URL ArrowReaderDatasetNotOnHfGcsErrorMissingFilesOnHfGcsErrorReadInstruction) ArrowWriter BeamWriter ParquetWriterSchemaInferenceError) DataFilesDictsanitize_patterns) DatasetDictIterableDatasetDict)DownloadConfig)DownloadManager DownloadMode)MockDownloadManager)StreamingDownloadManagerxopen)Features)is_remote_filesystemrename)Hasher) DatasetInfoDatasetInfosDictPostProcessedInfo)ExamplesIterableIterableDataset&_generate_examples_from_tables_wrapper)DuplicatedKeysError)"INVALID_WINDOWS_CHARACTERS_IN_PATHcamelcase_to_snakecase)Split SplitDictSplitGenerator SplitInfo)$extend_dataset_builder_for_streaming)logging) cached_path is_remote_url)FileLock)VerificationModeget_size_checksum_dictverify_checksums verify_splits) classpropertyconvert_file_size_to_inthas_sufficient_disk_spaceiflatmap_unordered map_nestedmemoizesize_strtemporary_assignment)_number_of_shards_in_gen_kwargs_split_gen_kwargsceZdZdS)InvalidConfigNameN__name__ __module__ __qualname__0lib/python3.11/site-packages/datasets/builder.pyrJrJWDrPrJceZdZdS)DatasetBuildErrorNrKrOrPrQrTrT[rRrPrTceZdZdS)ManualDownloadErrorNrKrOrPrQrVrV_rRrPrVceZdZdS)DatasetGenerationErrorNrKrOrPrQrXrXcrRrPrXceZdZUdZdZeed<ejdZ e e ejefed<dZ e eed<dZ e eed<dZe eed <d Zd Z dd ed e edefdZdS) BuilderConfigaBase class for `DatasetBuilder` data configuration. `DatasetBuilder` subclasses with data configuration options should subclass `BuilderConfig` and add their own properties. Attributes: name (`str`, defaults to `default`): version (`Version` or `str`, *optional*): data_dir (`str`, *optional*): data_files (`str` or `Sequence` or `Mapping`, *optional*): Path(s) to source data file(s). description (`str`, *optional*): defaultnamez0.0.0versionNdata_dir data_files descriptionctD]+}||jvr tdtd|jd,|j1t |jt st d|jdSdS)Nz Bad characters from black list 'z ' found in 'z\'. They could create issues when creating a directory for this config on Windows filesystem.z/Expected a DataFilesDict in data_files but got )r0r\rJr_ isinstancer ValueError)self invalid_chars rQ __post_init__zBuilderConfig.__post_init__}s>  Lty(('q7Yqqgkgpqqq) ? &z$/=/Y/Y &`t``aa a ' & & &rPc tjtjkrdStfdjDS)NFc3dK|]*}|t|f|t|fkV+dSN)getattr).0kords rQ z'BuilderConfig.__eq__..sD]]1AwtQ''(Q1 ,>>]]]]]]rP)set__dict__keysall)rdrms``rQ__eq__zBuilderConfig.__eq__st t}!!## $ $AJOO,=,=(>(> > >5]]]]] HZHZH\H\]]]]]]rP config_kwargscustom_featuresreturnc~d}|dddddvrdddrfdtDtdDrYddD}t|d krtj }ntj }|Nt}|r| || || }|rK|j d z|z}t|tjkr|j d ztj |z}|S|j S) a0 The config id is used to build the cache directory. By default it is equal to the config name. However the name of a config is not sufficient to have a unique identifier for the dataset being generated since it doesn't take into account: - the config kwargs that can be used to overwrite attributes - the custom features used to write the dataset - the data_files for json/text/csv/pandas datasets Therefore the config id is just the config name with an optional suffix based on these. Nr\r]r^c"i|] }|| SrOrO)rkrlconfig_kwargs_to_add_to_suffixs rQ z2BuilderConfig.create_config_id..s....9:1!4...rPc3fK|],}t|ttttfV-dSri)rbstrboolintfloat)rkvs rQrnz1BuilderConfig.create_config_id..s5kka:a#tS%!899kkkkkkrP,c3K|]F\}}t|dztjt|zVGdS)=N)r|urllibparse quote_plus)rkrlrs rQrnz1BuilderConfig.create_config_id..s\""GKq!CFFSL6<#:#:3q66#B#BB""""""rP -)copypopsortedrrvaluesjoinitemslenr(hashupdate hexdigestr\r%MAX_DATASET_CONFIG_ID_READABLE_LENGTH)rdrtrusuffixm config_idrys @rQcreate_config_idzBuilderConfig.create_config_ids"!%)6););)=)=&&**64888&**9d;;; 7 7 7DEc>d>d... *kkCaChChCjCjkkkkk E""OmOsOsOuOu"""v;;###[)GHHF%CDD  &A !   HH_ % % %[[]]F   C&0I9~~ LLL IOfk&.A.AA  9 rPri)rLrMrN__doc__r\r|__annotations__rVersionr]r r r^r_rr`rfrsdictr%rrOrPrQrZrZgs  D#3@5=3I3IGXeEM3./ 0III"Hhsm"""*.J'...!%K#%%% b b b^^^/3777"(+7 777777rPrZceZdZdZdZeZgZdZdZ dIde e de e de e de e de e d e e d e eee fd e e d e ee eeefd e e de ede efdZdZdZede e fdZedefdZde fdZ dJdeee ffdZeee dZ!edZ"dKde fdZ#dZ$e%j&de fdZ'edZ(de d e fd!Z) dLd#e e d$e e*d%e ee+e fd&e ee,e fd'ed(e e-de e d)e d*e eee fd+e ede efd,Z.d-Z/d$e*fd.Z0d/Z1d0Z2de fd1Z3d2Z4d3Z5d4Z6 dMd6e e7d&e ee,e fdee8e9ffd7Z: dNd6ee e;e7fd8ed&e,d9efd:Zd6ee;e7fde fd<Z? dJd6e e de e dee@e eAfeAffd=ZBdeAfd>ZCd?e8d@eDe e fde e8fdAZEd6e de@e e ffdBZFd6e dCe d(e-de e fdDZGe%j&d(e-fdEZHe%j& dOdFeId)e d*e ee efd+e efdGZJdFeIdeKfdHZLdS)PDatasetBuilderaAbstract base class for all datasets. `DatasetBuilder` has 3 key methods: - [`DatasetBuilder.info`]: Documents the dataset, including feature names, types, shapes, version, splits, citation, etc. - [`DatasetBuilder.download_and_prepare`]: Downloads the source data and writes it to disk. - [`DatasetBuilder.as_dataset`]: Generates a [`Dataset`]. Some `DatasetBuilder`s expose multiple variants of the dataset by defining a [`BuilderConfig`] subclass and accepting a config object (or name) on construction. Configurable datasets expose a pre-defined set of configurations in [`DatasetBuilder.builder_configs`]. Args: cache_dir (`str`, *optional*): Directory to cache data. Defaults to `"~/.cache/huggingface/datasets"`. config_name (`str`, *optional*): Name of the dataset configuration. It affects the data generated on disk. Different configurations will have their own subdirectories and versions. If not provided, the default configuration is used (if it exists). Parameter `name` was renamed to `config_name`. hash (`str`, *optional*): Hash specific to the dataset code. Used to update the caching directory when the dataset loading script code is updated (to avoid reusing old data). The typical caching directory (defined in `self._relative_data_dir`) is `name/version/hash/`. base_path (`str`, *optional*): Base path for relative paths that are used to download files. This can be a remote URL. features ([`Features`], *optional*): Features types to use with this dataset. It can be used to change the [`Features`] types of a dataset, for example. use_auth_token (`str` or `bool`, *optional*): String or boolean to use as Bearer token for remote files on the Datasets Hub. If `True`, will get token from `"~/.huggingface"`. repo_id (`str`, *optional*): ID of the dataset repository. Used to distinguish builders with the same name but not coming from the same namespace, for example "squad" and "lhoestq/squad" repo IDs. In the latter, the builder name would be "lhoestq___squad". data_files (`str` or `Sequence` or `Mapping`, *optional*): Path(s) to source data file(s). For builders like "csv" or "json" that need the user to specify data files. They can be either local or remote files. For convenience, you can use a `DataFilesDict`. data_dir (`str`, *optional*): Path to directory containing source data file(s). Use only if `data_files` is not passed, in which case it is equivalent to passing `os.path.join(data_dir, "**")` as `data_files`. For builders that require manual download, it must be the path to the local directory containing the manually downloaded data. storage_options (`dict`, *optional*): Key/value pairs to be passed on to the dataset file-system backend, if any. writer_batch_size (`int`, *optional*): Batch size used by the ArrowWriter. It defines the number of samples that are kept in memory before writing them and also the length of the arrow chunks. None means that the ArrowWriter will use its default value. name (`str`): Configuration name for the dataset. Use `config_name` instead. **config_kwargs (additional keyword arguments): Keyword arguments to be passed to the corresponding builder configuration class, set on the class attribute [`DatasetBuilder.BUILDER_CONFIG_CLASS`]. The builder configuration class is [`BuilderConfig`] or a subclass of it. N deprecated cache_dir config_namer base_pathinfofeaturesuse_auth_tokenrepo_idr_r^storage_optionswriter_batch_sizec ^ | dkrtjdt| }t|jdd|_||_||_||_ ||_ | |_ | p|j |_ | 9t| ts$tjt#| ||} dt%j|jjjvr|||d<| | |d<| | |d <|jd||d |\|_|_|i|}|||j|_|jj|_|jj|_||_ | ||j _!tE|p t0j#|_$tK|j$r|j$n#tLj'(|j$|_$tK|j$r tRj*ntLj'j*}|r||j$t0j+ntEt0j,|_-tK|j-r|j-n#tLj'(|j-|_-|.|_/tK|j$stMj0|j$d tLj'*|j$|j/1tLj2d dz}tg|5tLj'4|j/rtktMj6|j/dkrqtLj'4||j/t0j7r8tp dtsj:|j/|_ nDtp;d|j/d|jdtMj<|j/dddn #1swxYwY|j/|_=t}j?d|_@d|_Ad|_Bt|dS)Nrz\Parameter 'name' was renamed to 'config_name' in version 2.3.0 and will be removed in 3.0.0.)category.)rrrr_r^)rruTexist_ok_z.lockrz?? " //11T122 2 K,t < < < < T%94?;R;RSUSY[^;_;_bi;ijjI)$$ 2 27>>$/22 22:do6677!;;7>>))DOVEa*b*bccT"KK(fggg(3(B4?(S(SDII$/IIPTPYIII111 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 ?.4.?.G.G# -T22222sDQ  Q$'Q$c|jSri)rprds rQ __getstate__zDatasetBuilder.__getstate__s }rPc2||_t|dSri)rpr6)rdds rQ __setstate__zDatasetBuilder.__setstate__s ,T22222rPrvcdSrirOrs rQmanual_download_instructionsz+DatasetBuilder.manual_download_instructionstrPcNtj|S)a"Empty dict if doesn't exist Example: ```py >>> from datasets import load_dataset_builder >>> ds_builder = load_dataset_builder('rotten_tomatoes') >>> ds_builder.get_all_exported_dataset_infos() {'default': DatasetInfo(description="Movie Review Dataset. This is a dataset of containing 5,331 positive and 5,331 negative processed sentences from Rotten Tomatoes movie reviews. This data was first used in Bo Pang and Lillian Lee, ``Seeing stars: Exploiting class relationships for sentiment categorization with respect to rating scales.'', Proceedings of the ACL, 2005. ", citation='@InProceedings{Pang+Lee:05a, author = {Bo Pang and Lillian Lee}, title = {Seeing stars: Exploiting class relationships for sentiment categorization with respect to rating scales}, booktitle = {Proceedings of the ACL}, year = 2005 } ', homepage='http://www.cs.cornell.edu/people/pabo/movie-review-data/', license='', features={'text': Value(dtype='string', id=None), 'label': ClassLabel(num_classes=2, names=['neg', 'pos'], id=None)}, post_processed=None, supervised_keys=SupervisedKeysData(input='', output=''), task_templates=[TextClassification(task='text-classification', text_column='text', label_column='label')], builder_name='rotten_tomatoes_movie_review', config_name='default', version=1.0.0, splits={'train': SplitInfo(name='train', num_bytes=1074810, num_examples=8530, dataset_name='rotten_tomatoes_movie_review'), 'validation': SplitInfo(name='validation', num_bytes=134679, num_examples=1066, dataset_name='rotten_tomatoes_movie_review'), 'test': SplitInfo(name='test', num_bytes=135972, num_examples=1066, dataset_name='rotten_tomatoes_movie_review')}, download_checksums={'https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz': {'num_bytes': 487770, 'checksum': 'a05befe52aafda71d458d188a1c54506a998b1308613ba76bbda2e5029409ce9'}}, download_size=487770, post_processing_size=None, dataset_size=1345461, size_in_bytes=1833231)} ``` )r*rget_imported_module_dirclss rQget_all_exported_dataset_infosz-DatasetBuilder.get_all_exported_dataset_infoss! .s/J/J/L/LMMMrPc~||jjt S)aEmpty `DatasetInfo` if doesn't exist Example: ```py >>> from datasets import load_dataset_builder >>> ds_builder = load_dataset_builder('rotten_tomatoes') >>> ds_builder.get_exported_dataset_info() DatasetInfo(description="Movie Review Dataset. This is a dataset of containing 5,331 positive and 5,331 negative processed sentences from Rotten Tomatoes movie reviews. This data was first used in Bo Pang and Lillian Lee, ``Seeing stars: Exploiting class relationships for sentiment categorization with respect to rating scales.'', Proceedings of the ACL, 2005. ", citation='@InProceedings{Pang+Lee:05a, author = {Bo Pang and Lillian Lee}, title = {Seeing stars: Exploiting class relationships for sentiment categorization with respect to rating scales}, booktitle = {Proceedings of the ACL}, year = 2005 } ', homepage='http://www.cs.cornell.edu/people/pabo/movie-review-data/', license='', features={'text': Value(dtype='string', id=None), 'label': ClassLabel(num_classes=2, names=['neg', 'pos'], id=None)}, post_processed=None, supervised_keys=SupervisedKeysData(input='', output=''), task_templates=[TextClassification(task='text-classification', text_column='text', label_column='label')], builder_name='rotten_tomatoes_movie_review', config_name='default', version=1.0.0, splits={'train': SplitInfo(name='train', num_bytes=1074810, num_examples=8530, dataset_name='rotten_tomatoes_movie_review'), 'validation': SplitInfo(name='validation', num_bytes=134679, num_examples=1066, dataset_name='rotten_tomatoes_movie_review'), 'test': SplitInfo(name='test', num_bytes=135972, num_examples=1066, dataset_name='rotten_tomatoes_movie_review')}, download_checksums={'https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz': {'num_bytes': 487770, 'checksum': 'a05befe52aafda71d458d188a1c54506a998b1308613ba76bbda2e5029409ce9'}}, download_size=487770, post_processing_size=None, dataset_size=1345461, size_in_bytes=1833231) ``` )rgetrr\r)rs rQrz(DatasetBuilder.get_exported_dataset_infos/2244889I;==YYYrPc &d}||jr|s|jJ|j|j}td|jd|jnt|jdkr[d|jd|jdjd}td t|j d |d z|jd}t d |jd|jt|tr\|j|}|@|jr9td |dt|j |sN|||d<n|jr |s |j|d<d|vr!t|dr|jr |j|d<|jdi|}ngt#j|}|D]>\}}|7t||std |d|dt)|||?|jstd|j|||}||jvo|dk} | rt d|ny|j|jvrL||j|jkr6tdt|j |jstd |jd||fS)aCreate and validate BuilderConfig object as well as a unique config id for this config. Raises ValueError if there are multiple builder configs and config_name and DEFAULT_CONFIG_NAME are None. config_kwargs override the defaults kwargs in config Nz$No config specified, defaulting to: /rload_dataset('', 'rz')zEConfig name is missing. Please pick one among the available configs: z Example of usage: ``z6No config specified, defaulting to the single config: zBuilderConfig z not found. Available: r\r]VERSIONz doesn't have a 'z' key.z$BuilderConfig must have a name, got )rur[z Using custom data configuration zvCannot name a custom BuilderConfig the same as an available BuilderConfig. Change the name. Available BuilderConfigs: z must have a versionrO)BUILDER_CONFIGSDEFAULT_CONFIG_NAMEbuilder_configsrrrr\rrclistrqrrbr|hasattrrrrdeepcopyrsetattrrr]) rdrrurtbuilder_configexample_of_usagekeyvaluer is_customs rQrz%DatasetBuilder._create_builder_configs  4#7  '3!%!5!9!9$:R!S!SgdiggR`Regghhhht+,,q00'g 'g'gtG[\]G^Gc'g'g'g$$nJNtOcOhOhOjOjJkJknnH5EHHHI "&!5a!8 vUYU^vvaoatvvwww k3 ' ' !155kBBN%$*>% l[lldNbNgNgNiNiIjIjll  8&(3 f%%) A- A(,(@ f% --'$ 2J2J-t|-+/< i(6T6GGGGNN"]>::N+1133 8 8 U$">377h()f.)f)f[^)f)f)fgggNC777" [YNDWYYZZ Z#33 +4  d&::V Y@V  ] KKF9FF G G G G#t';;;"d&:>;N&OOO uQUVZVjVoVoVqVqQrQruu") ] ![.2E![![![\\\y((rPcd|jD}t|t|jkr#d|jD}td||S)z:Pre-defined list of configurations for this builder class.ci|] }|j| SrOr\rkrs rQrzz2DatasetBuilder.builder_configs..sIII66;IIIrPcg|] }|j SrOrrs rQ z2DatasetBuilder.builder_configs..sCCCVV[CCCrPz5Names in BUILDER_CONFIGS must not be duplicated. Got )rrrc)rconfigsnamess rQrzDatasetBuilder.builder_configssj JIS5HIII w<<3s233 3 3CCs/BCCCE\UZ\\]] ]rPc|jSri)rrs rQrzDatasetBuilder.cache_dirs rPTc|jr>|jddkr |jddnd}||jn |d|j}|j}|j}|rt jjn tj}|r|||j }|r#||t|jj }|r#|r!t|tr |||}|S)aoRelative path of this dataset in cache_dir: Will be: self.name/self.config.version/self.hash/ or if a repo_id with a namespace has been specified: self.namespace___self.name/self.config.version/self.hash/ If any of these element is missing or if ``with_version=False`` the corresponding subfolders are dropped. rrN___)rcountrr\rrrrrrrr|r]rb) rd with_version with_hashis_local namespacebuilder_data_dirrrrs rQ_relative_data_dirz!DatasetBuilder._relative_data_dir#s37,h4JJ   U(y)93t{?R;S;STT   A A*T3"7"7 A(y)94@@ rPc 2t|j }|rtjjn t j}||j|d|||j|d|}fd}ts|}|ru|dd}||jjkrWdt|d|j d|jd t|jjd }t ||S) z2Return the data directory for the current version.F)r r TctjsgSg}tjD]<} |t j||f-#t$rY9wxYw|d|S)z"Returns previous versions on disk.T)reverse) rrrrappendrrrcsort)version_dirnamesdir_namers rQ_other_versions_on_diskz@DatasetBuilder._build_cache_dir.._other_versions_on_diskDs7>>"233  ! J'788  $++U]8-D-Dh,OPPPP!D  ! !$ ! / / /# #s)A&& A32A3rzFound a different version z of dataset z in cache_dir z". Using currently defined version r) r9rrrrrrrr]r|r\rr) rdr rversion_data_dirr version_dirs other_versionwarn_msgrs @rQrzDatasetBuilder._build_cache_dir9s_$T%9:::$,@BGLL). $9  $"9"9uW_"9"`"`  %9  $"9"9tV^"9"_"_   $ $ $ $ $-.. -2244L - ,Q 2  DK$7777S5G5G77UYU^77%)%977t{233777 NN8,,,rPct)a Construct the DatasetInfo object. See `DatasetInfo` for details. Warning: This function is only called once and the result is cached for all following .info() calls. Returns: info: (DatasetInfo) The dataset information NotImplementedErrorrs rQrzDatasetBuilder._infoas "!rPctjtjtj|S)z8Return the path of the module of this class or subclass.)rrdirnamergetfile getmodulers rQrz&DatasetBuilder.get_imported_module_dirms-www/@/E/EFFGGGrPsrcdstc2t|j||dSri)r'r)rdr$r%s rQ_renamezDatasetBuilder._renamerstxc"""""rParrow output_dirdownload_config download_modeverification_modetry_from_hf_gcsr file_formatmax_shard_sizenum_procc n|dkr=|r tjn tj}tjd|jdt | dkrtjdt nj} ||nj}tj || }|d_ tj  r|ddn%j |dd_t|p tj}t|p tj}||nj}| | d vrt'd | d j jd kr(t+d jdjjzd|v|||j5}td'|5d}|rt @|jAd}nV#ttf$rt\1d(Yn)t$rt\/d)YnwxYw|s$d*| i}| | |d+<| | |d,<jEd2||d-||td.j1jGHDj1_8|Ij1_Jj1j8j1j7zj1_4Kdddn #1swxYwYdddn #1swxYwY2|twd/jd0jd1ddddS#1swxYwYdS)3aDownloads and prepares dataset for reading. Args: output_dir (`str`, *optional*): Output directory for the dataset. Default to this builder's `cache_dir`, which is inside `~/.cache/huggingface/datasets` by default. download_config (`DownloadConfig`, *optional*): Specific download configuration parameters. download_mode ([`DownloadMode`] or `str`, *optional*): Select the download/generate mode, default to `REUSE_DATASET_IF_EXISTS`. verification_mode ([`VerificationMode`] or `str`, defaults to `BASIC_CHECKS`): Verification mode determining the checks to run on the downloaded/processed dataset information (checksums/size/splits/...). ignore_verifications (`bool`, defaults to `False`): Ignore the verifications of the downloaded/processed dataset information (checksums/size/splits/...). `ignore_verifications` was deprecated in version 2.9.1 and will be removed in 3.0.0. Please use `verification_mode` instead. try_from_hf_gcs (`bool`): If `True`, it will try to download the already prepared dataset from the HF Google cloud storage. dl_manager (`DownloadManager`, *optional*): Specific `DownloadManger` to use. base_path (`str`, *optional*): Base path for relative paths that are used to download files. This can be a remote url. If not specified, the value of the `base_path` attribute (`self.base_path`) will be used instead. use_auth_token (`Union[str, bool]`, *optional*): Optional string or boolean to use as Bearer token for remote files on the Datasets Hub. If True, or not specified, will get token from ~/.huggingface. Pass `use_auth_token` to the initializer/`load_dataset_builder` instead. file_format (`str`, *optional*): Format of the data files in which the dataset will be written. Supported formats: "arrow", "parquet". Default to "arrow" format. If the format is "parquet", then image and audio data are embedded into the Parquet files instead of pointing to local files. max_shard_size (`Union[str, int]`, *optional*): Maximum number of bytes written per shard, default is "500MB". The size is based on uncompressed data size, so in practice your shard files may be smaller than `max_shard_size` thanks to Parquet compression for example. num_proc (`int`, *optional*, defaults to `None`): Number of processes when downloading and generating the dataset locally. Multiprocessing is disabled by default. storage_options (`dict`, *optional*): Key/value pairs to be passed on to the caching file-system backend, if any. **download_and_prepare_kwargs (additional keyword arguments): Keyword arguments. Example: Download and prepare the dataset as Arrow files that can be loaded as a Dataset using `builder.as_dataset()`: ```py >>> from datasets import load_dataset_builder >>> builder = load_dataset_builder("rotten_tomatoes") >>> ds = builder.download_and_prepare() ``` Download and prepare the dataset as sharded Parquet files locally: ```py >>> from datasets import load_dataset_builder >>> builder = load_dataset_builder("rotten_tomatoes") >>> ds = builder.download_and_prepare("./output_dir", file_format="parquet") ``` Download and prepare the dataset as sharded Parquet files in a cloud storage: ```py >>> from datasets import load_dataset_builder >>> storage_options = {"key": aws_access_key_id, "secret": aws_secret_access_key} >>> builder = load_dataset_builder("rotten_tomatoes") >>> ds = builder.download_and_prepare("s3://my-bucket/my_rotten_tomatoes", storage_options=storage_options, file_format="parquet") ``` rz'ignore_verifications' was deprecated in favor of 'verification_mode' in version 2.9.1 and will be removed in 3.0.0. You can remove this warning by passing 'verification_mode= ' instead.z'use_auth_token' was deprecated in version 2.7.1 and will be removed in 3.0.0. Pass `use_auth_token` to the initializer/`load_dataset_builder` instead.Nrr)r(parquetzUnsupported file_format: z. Expected 'arrow' or 'parquet'z7Unable to download and prepare the dataset at the root z'. Please specify a subdirectory, e.g. ''F)rforce_download force_extractuse_etagr0rr) dataset_namer*r^rrecord_checksumsr(T)parentsrz _builder.lockzFound cached dataset z ()zGenerating dataset ) directoryzNot enough disk space. Needed: z (download: z , generated: z, post-processed: c3Ks"j|d|VdS|dz}tj|d |Vtj|rt j|t j||tj|rt j|dSdS#tj|rt j|wwxYw)z4Create temporary dir for dirname and rename on exit.Trz .incompleteN) rrrrisdirshutilrmtreemover)r!tmp_dirr rds rQincomplete_dirz;DatasetBuilder.download_and_prepare..incomplete_dir@s 3H%%g%===!MMMMM% 5GK$77773% 7==113"M'222 GW5557>>'223"M'222223327>>'223"M'22223s A C6C=z"Downloading and preparing dataset rz , total: z) to z... to rzJDataset not on Hf google storage. Downloading and preparing it from sourcezGHF google storage unreachable. Downloading and preparing it from sourcer.r/r0)rr,c3$K|] }|jV dSri) num_bytes)rkrs rQrnz6DatasetBuilder.download_and_prepare..s$0h0hU0h0h0h0h0h0hrPDataset z downloaded and prepared to z(. Subsequent calls will reuse this data.rO)Lr; NO_CHECKS ALL_CHECKSrrrrrrrget_fs_token_pathsrr&unstrip_protocolrr!REUSE_DATASET_IF_EXISTS BASIC_CHECKSrrc_strip_protocol RuntimeErrorr\rrFORCE_REDOWNLOADrr rr^rrbr"rrparentmkdirr: contextlib nullcontextrrrrrrrr _load_infor"download_post_processing_resourcesrA size_in_bytesOSErrorrE download_size dataset_sizepost_processing_sizecontextmanagerprintrr_check_manual_downloadrF_download_prepared_from_hf_gcsr*rrConnectionError_download_and_preparesumsplitsrget_recorded_sizes_checksumsdownload_checksums _save_info)rdr)r*r+r,ignore_verificationsr-rrrr.r/r0rdownload_and_prepare_kwargsfs_token_pathsrr data_existsrF_desttmp_output_dirdownloaded_from_gcsprepare_split_kwargsr s` @rQdownload_and_preparez#DatasetBuilder.download_and_prepareus X < / />R s 0 : :XhXs  MqM^Mdqqq    \ ) ) Mj     "0N#-#9ZZt 2:___.!,Q//AZAZ[ijk[lmn[oApAp$]%Zl6Z[[ ,->-_BRB_``!*!6IIDN  "{:N'N'Neeeeff f 8 # #D$4 5 5 ; ;X$JZXX8<8H498TXXX   &"0"8#0L4Q#Q"/<3P"P"%#1$($8###)!Y /-#"&"4"h8IM]Mh8h J z#6 7 7 $ $g%%)#O$  ; ! " " ) / /t / L L L(?:I%- JXi *2H2J2Jb b (0D inI(//))D4DfFb*c*cddK } 0TTTWtyWWDDTWWWXXX!OO-- 77 CCCb b b b b b b b  KKNdiNN4;KNNN O O O 0I+0qDAQ??\\GKFV\\\GOd001ABBBTXTdw9OwwRVR[Rgwwmrwww  ' ' 3 3 3  011 &^*$ ~NN&&*/'&vv ?? @Z[[[26// 68PQvvv"KK(tuuuuu.vvv"NN+tuuuuuv. 0={/K,)5ES01AB#/?G0<22'1.?3: .10h0hdiN^NeNeNgNg0h0h0h-h-hDI*3=3Z3Z3\3\DI0.2i.DtyG^.^DI+OO%%%9&&&&&&&&&&&&&&& & & & & & & & & & & & & & & &D  3 3J ? ? ? :49::$BR:::   b b b b b b b b b b b b b b b b b b sB?\*H\*5[[ V*)[*+W= [#W= :[<W= =B=[: [[ [ [ [ \*[! !\*$[! %8\**\.1\.c |jN|jIttjd|jd|jjd|jd|jd dSdS)Nz The dataset z with config zp requires manual data. Please follow the manual download instructions: za Manual data can be loaded with: datasets.load_dataset("z$", data_dir=""))r manual_dirrVtextwrapdedentr\rrdrs rQraz%DatasetBuilder._check_manual_downloads  , 8Z=R=Z%_!%__9=9I__7__ .2Y ___    9 8=Z=ZrPc .|dd}t|j|j}|||t j|j}|j|tdz| tj dz}|jj D]}| |D]}tj |vrtd| t!|dz|z}t#j|tj|j|q#t*$r!t,d|dYwxYwt,ddS) NTF)r r r+Resources shouldn't be in a sub-directory: z Couldn't download resourse file z from Hf google storage.z*Dataset downloaded from Hf google storage.)rrrrdownload_from_hf_gcsr)rrrrrrrf_post_processing_resourcesrrcr8rBrDrrrcr) rdr*relative_data_dirreaderdownloaded_inforemote_cache_dirrresource_file_name resource_paths rQrbz-DatasetBuilder._download_prepared_from_hf_gcss 33QV3WWT-ty99##O5FGGG%4T5EFF )))*S03D3L3LRVUX3Y3YYY% q qE&*&E&Ee&L&L&S&S&U&U q q"6///$%gSe%g%ghhhq$/0@30FI[0[$\$\MK rw||DA E  (E76E7c t|j}||}|j|fi|}|tjkr4|jr-t|jj | d|D]}t|j j dkrtdtd|j jd||j  |j|fi|nr#t$$r2}t%d|jpdzd zt|zd d }~wt($r+}t)|j|jd |jd  d d }~wwxYw||tjks|tjkrt3|jj|||j_|j|j_d S)aDownloads and prepares dataset for reading. This is the internal implementation to overwrite called when user calls `download_and_prepare`. It should download all required data and generate the pre-processed datasets files. Args: dl_manager ([`DownloadManager`]): `DownloadManager` used to download and cache data. verification_mode ([`VerificationMode`]): if `ALL_CHECKS`, perform all the verifications including checksums. if `BASIC_CHECKS`, do not perform checksums, only perform split tests. if `NO_CHECKS`, do not perform any verification. prepare_split_kwargs: Additional options, such as `file_format`, `max_shard_size` )r;zdataset source filesrrz{`all` is a special split keyword corresponding to the union of all splits, so cannot be used as key in ._split_generator(). Generating  splitzCannot find data file. r6z Original error: Nz7To avoid duplicate keys, please fix the dataset script z.py)fix_msg)r3r\_make_split_generators_kwargs_split_generatorsr;rLr<r=rrhrgr| split_infolowerrcradd_prepare_splitr[rr/rduplicate_key_indicesmanage_extracted_filesrPr>rfdownloaded_sizer\) rdrr,rq split_dictsplit_generators_kwargssplit_generatorssplit_generatores rQrdz$DatasetBuilder._download_and_preparesS"DI666 "&"D"DEY"Z"Z141*XX@WXX  0 ; ; ; @[ ;  ,j.U.U.W.WYo    0 0 0O?-23399;;uDD + KKMo&@&EMMM N N N NN?5 6 6 6 ##OLL7KLLLL   -8>B@+,!ff  '   )E+dVZV_ddd    - - / / / /  0 = = =ARVfVqAqAq $)*J 7 7 7& ","< s$D F-E  F&E<<Fc@|jjpgD] }||D]\}}t |jrt d|jtj|vrtd|tj |j |}tj |sN||||}|r5td|d|t!j||dS)Nz/Post processing is not supported on filesystem ryz$Downloaded post-processing resource z as )rrfr{rr&rrrrrcrrrr#_download_post_processing_resourcesrrBrD)rdrr resource_namerrdownloaded_resource_paths rQrYz1DatasetBuilder.download_post_processing_resourcessEY%+ M ME595T5TUZ5[5[5a5a5c5c M M1 1/99l-.j`d`h.j.jkkk6///$%gSe%g%ghhh " T-=?Q R R w~~m44M/3/W/W}j00,0M $r=$r$r^p$r$rsss $ ignore_verifications (`bool`, defaults to `False`): Whether to ignore the verifications of the downloaded/processed dataset information (checksums/size/splits/...). `ignore_verifications` was deprecated in version 2.9.1 and will be removed in 3.0.0. Please use `verification_mode` instead. in_memory (`bool`, defaults to `False`): Whether to copy the data in-memory. Returns: datasets.Dataset Example: ```py >>> from datasets import load_dataset_builder >>> builder = load_dataset_builder('rotten_tomatoes') >>> ds = builder.download_and_prepare() >>> ds = builder.as_dataset(split='train') >>> ds Dataset({ features: ['text', 'label'], num_rows: 8530 }) ``` rz'ignore_verifications' was deprecated in favor of 'verification' in version 2.9.1 and will be removed in 3.0.0. You can remove this warning by passing 'verification_mode=r2zLoading a dataset cached in a z is not supported.rJz: could not find data in z. Please make sure to call builder.download_and_prepare(), or use datasets.load_dataset() before trying to access the Dataset object.zConstructing Dataset for split z, z, from Nci|]}||SrOrO)rkss rQrzz-DatasetBuilder.as_dataset.._s444aQ444rP)run_post_processr, in_memoryT) map_tuple disable_tqdm) rKr;rLrrrrr&rrtyperLrrrrFileNotFoundErrorr\rdebugrrrfrPrCr_build_single_datasetr7is_progress_bar_enabledrbrr)rdrrr,rjrr datasetss rQ as_datasetzDatasetBuilder.as_datasetsf < / /?S t 1 ; ;YiYt  MqM^Mdqqq    ,DH555 t%&rtDH~~G^&r&r&rss sw~~d.// #V49VVt?OVVV   vu7[ $)JZ@[@[vvdhdtvvwww =4449#3444E,->-_BRB_`` *!1"3#     $<>>>     h % % -"8,,HrPrrct|tsUt|}|dkr1djj}t|}||}|r | D]"}tj |vrtd|#fd |D}||}|]|}i} d} |D]\} } t!| } | | | <|t"jkrW| rUjjjjjd}n$jjj|}t-|| djjt/j_jjjijj_| jjjt|<t1d jjj Dj_jj<jj0jjjjzjjzj_jj|j_jj|j_jj|j_jjjhjjjj |jj kr)td jjjd |jjjj|j_|S) zas_dataset for a single split.rr+)rrrycbi|]+\}}|tjj|,SrO)rrrr)rkrrrds rQrzz8DatasetBuilder._build_single_dataset..sD5M#5rw||D,<>PQQrPNFzpost processing resourcesc3TK|]#}|D] }|dV $dS)rIN)r)rksplit_checksums_dictschecksums_dicts rQrnz7DatasetBuilder._build_single_dataset..s^55-*?*F*F*H*H55'#;/5555555rPz:Post-processed features info don't match the dataset: Got z but expected something like )!rbrr|rrrfrqr2 _as_datasetr{rrrrcr _post_processr<r;rLpost_processedresources_checksumsrr=r+rer^r]r\rZrirrr)rdrrr,rdsrresources_pathsrrecorded_checksumsr<rr size_checksumexpected_checksumss` rQrz$DatasetBuilder._build_single_datasetss%11 !JJE~~!1!6!6!8!899%LLE     . M&*&E&Ee&L&L&S&S&U&U i i"6///$%gSe%g%ghhh09=9X9XY^9_9_9e9e9g9gO"//ODDN)#%'"#( 4C4I4I4K4KFF0M=$:=$I$IM8E&}55$(8(CCCHXCy/749;S;g;o-1**-1Y-E-Y-]-]^c-d-d*$%79KMhiii9+3/@/B/BDI,9+?GCEDI,@K] (> $Y ::?? (//11 @   33E:: AA;A.AAArPct}||tjd|t ||}|S)zThe dataset fingerprint is the hash of the relative directory dataset_name/config_name/version/hash, as well as the split specs.r)r(rrrrrr|r)rdrhasherrs rQrz'DatasetBuilder._get_dataset_fingerprintsj d--//77DDEEE c%jj!!!&&(( rPcRt|j }|s*tdt|jjdt |p|jt|j|j |j |j j }| |d||D}||}n/||vr ||}n"td|dt!|t#|j|d }t'|t(rt+|}|S) Nz(Loading a streaming dataset cached in a z is not supported yet.)rr)rr*r;r^ci|] }|j| SrOr)rksgs rQrzz7DatasetBuilder.as_streaming_dataset..sVVVRRWbVVVrP Bad split: . Available splits: T)r)r&rrrrLr#rrrrr\rr^rarrcrrC_as_streaming_dataset_singlerbrr)rdrrr rsplits_generatorssplits_generatorrs rQas_streaming_datasetz#DatasetBuilder.as_streaming_datasets] ,DH555 %j4>>;Rjjj .14>*$:M_c_sttt[)    ##J///VV43I3I*3U3UVVV =0   ' ' '07  _5__dK\F]F]__`` `  -     h % % 5*844HrPc||}|jr|j|jini}t||j|j|S)N)rrtoken_per_repo_id) _get_examples_iterable_for_splitrrr-rr\)rdr ex_iterablers rQrz+DatasetBuilder._as_streaming_dataset_singles\;;??UW di/?/DXi    rPdatasetrcdS)z%Run dataset transforms or add indexesNrO)rdrrs rQrzDatasetBuilder._post_process rrPciS)z+Mapping resource_name -> resource_file_namerOrdrs rQr{z)DatasetBuilder._post_processing_resourcess rPrcdS)zPDownload the resource using the download manager and return the downloaded path.NrO)rdrrrs rQrz2DatasetBuilder._download_post_processing_resourcess trPct)aSpecify feature dictionary generators and dataset splits. This function returns a list of `SplitGenerator`s defining how to generate data and what splits to use. Example: return [ datasets.SplitGenerator( name=datasets.Split.TRAIN, gen_kwargs={'file': 'train_data.zip'}, ), datasets.SplitGenerator( name=datasets.Split.TEST, gen_kwargs={'file': 'test_data.zip'}, ), ] The above code will first call `_generate_examples(file='train_data.zip')` to write the train data, then `_generate_examples(file='test_data.zip')` to write the test data. Datasets are typically split into different subsets to be used at various stages of training and evaluation. Note that for datasets without a `VALIDATION` split, you can use a fraction of the `TRAIN` data for evaluation as you iterate on your model so as not to overfit to the `TEST` data. For downloads and extractions, use the given `download_manager`. Note that the `DownloadManager` caches downloads, so it is fine to have each generator attempt to download the source data. A good practice is to download all data in this function, and then distribute the relevant parts to each split with the `gen_kwargs` argument Args: dl_manager (`DownloadManager`): Download manager to download the data Returns: `list`. rrws rQrz DatasetBuilder._split_generatorssZ"###rPrc t)a!Generate the examples and record them on disk. Args: split_generator (`SplitGenerator`): Split generator to process file_format (`str`, *optional*): format of the data files in which the dataset will be written. Supported formats: "arrow", "parquet". Default to "arrow" format. max_shard_size (`Union[str, int]`, *optional*): Maximum number of bytes written per shard, default is "500MB". The size is based on uncompressed data size, so in practice your shard files may be smaller than `max_shard_size` thanks to Parquet compression for example. num_proc (`int`, *optional*, defaults to `None`): Number of processes when downloading and generating the dataset locally. Multiprocessing is disabled by default. **kwargs: Additional kwargs forwarded from _download_and_prepare (ex: beam pipeline) r)rdrr.r/r0kwargss rQrzDatasetBuilder._prepare_splitG:"###rPct)zGenerate the examples on the fly. Args: split_generator (`SplitGenerator`): Split generator to process rrdrs rQrz/DatasetBuilder._get_examples_iterable_for_splitfs"###rP) NNNNNNNNNNNNr)NN)TTT) NNNNrTNNrr(NNN)NTNrF)Fr(NN)MrLrMrNrrrZrrrrr r|r)r%r r}rrrr~rrrpropertyr classmethodr*rrr rr?rDrrrrabcabstractmethodrrr'rr!r;r rrrarbrdrYrXrirrr2rrrrrTRAINrrrr-rrrrr{rrr4rr,rrOrPrQrrsJJZG)O !%$(%)"#'&*'+59!%FJ"&*.+/ k3k3C=k3c]k3sm k3 C= k3 {# k38$k3!tSy!12k3#k3U3dM#ABCk33-k3"$k3$C=k3k3k3k3Z333hsmX N/? N N N[ N Z; Z Z Z Z15N)N) }c! "N)N)N)N)` WYYY[]X  VY    ,& & & P  "{ " " " "HH[H#3#S#### %)48<@DH) $04#'#"48"&*.XXSMX".1X lC&7 89 X $E*:C*?$@A XX_-XC=XX!sCx1X3-X"$XXXXt   BnBBBB(?=?=?=BMMM fKffffeeeqqq"&DH) ZZZ$E*:C*?$@A Z w # $ZZZZB CCS/501CC, C  CCCCJBG`eBB'=!>BY]BjqBBBB4eOU4J.KPS $#'##}#C=# tC()?: ; ####J        WwsCx?PU]^eUfS#X),:I #  ,$O,$,$,$,$\ #48"& $$'$$!sCx1 $ 3- $$$$<$$Sc$$$$$$rPrceZdZdZejdZ ddedede e de e e e ffd Z d ed e d e de d edede deee ee e efffdZfdZdedefdZxZS)GeneratorBasedBuilderawBase class for datasets with data generation based on dict generators. `GeneratorBasedBuilder` is a convenience class that abstracts away much of the data writing and reading of `DatasetBuilder`. It expects subclasses to implement generators of feature dictionaries across the dataset splits (`_split_generators`). See the method docstrings for details. c t)agDefault function generating examples for each `SplitGenerator`. This function preprocess the examples from the raw data to the preprocessed dataset files. This function is called once for each `SplitGenerator` defined in `_split_generators`. The examples yielded here will be written on disk. Args: **kwargs (additional keyword arguments): Arguments forwarded from the SplitGenerator.gen_kwargs Yields: key: `str` or `int`, a unique deterministic example identification key. * Unique: An error will be raised if two examples are yield with the same key. * Deterministic: When generating the dataset twice, the same example should have the same key. Good keys can be the image id, or line number if examples are extracted from a text file. The key will be hashed and sorted to shuffle examples deterministically, such as generating the dataset multiple times keep examples in the same order. example: `dict`, a feature dictionary ready to be encoded and written to disk. The example will be encoded with `self.info.features.encode_example({...})`. rrdrs rQ_generate_examplesz(GeneratorBasedBuilder._generate_examplesyrrPr(Nrcheck_duplicate_keysr0r/c \ !"t|p tj}tj }|rt jjn tj}j j j j |j }n|j }d} j d|j | d|} |j |  |r|dkr{t|j} | dkr+|)t d|d|j dd}n6|4| |kr.t d|d| d |j d | d | }t%jt%j d |jd d|j d} ||||d||dkrvd} |j}d}| 5jd(||dD] \}}}|r|} | |! dddn #1swxYwY| Jdd| D\}}}!}nfdt1t3|j|D}t5|}dg|z}dg|z}dg|z}dg|z!dg|z}t7|5}| 5t9|j|D]5\}}}|r|\||<||<||<!|<||< | |6 dddn #1swxYwYdddn #1swxYwYd|vsJd|dt;!"t;|}t;|}|d}||j _||j _t d"d "dkr^dt@tBf !"fd }dt1!D}tE||dd !d"|D|j _#n[d#\}}$ %d$|d%%d&|d% %| d'j j&|j _&dSdS))N-JJJJJ-SSSSS-of-NNNNNrrrSetting num_proc from  back to 1 for the @ split to disable multiprocessing as it only contains one shard.rG for the  split as it only contains  shards. examplesFrrdisableunittotalleavedesc)fpathr.r/rrr gen_kwargsjob_id-Failed to retrieve results from prepare_splitcg|]}|gSrOrOrkitems rQrz8GeneratorBasedBuilder._prepare_split..-hhhhhhrPc$g|] \}}||d SrrOrkrr_prepare_split_argss rQrz8GeneratorBasedBuilder._prepare_split..;&FJ *VSS?RSrP max_num_jobskwargs_iterable;Failed to retrieve results from prepare_split: result list G still contains None - at least one worker failed to return its results Renaming shard_and_jobc"|\}}td||z}d|dd|dd|ddddSNSSSSS05dJJJJJz JJJJJ-SSSSSNNNNNrer'r)r shard_idrglobal_shard_idrrdshards_per_job total_shardss rQ _rename_shardz;GeneratorBasedBuilder._prepare_split.._rename_shard s#0 &"%nWfW&=">">"I MM'h+<+<==EEgRX__MM-O1I1IJJRRSZ_k\q\qrrrPc@g|]\}}t|D]}||fSrOrangerkr num_shardsrs rQrz8GeneratorBasedBuilder._prepare_split..sP&FJ %j 1 16"rPT@r max_workerscg|] }|D]}| SrOrOrk shard_lengths shard_lengths rQrz8GeneratorBasedBuilder._prepare_split..:888!.\i88LX 8888rPrrrrrr6rO'r@rMAX_SHARD_SIZEr&rrrrrrrfr\rrrGrrrr7tqdmr num_examples_prepare_split_singler enumeraterHrr rBrerIrr r~r r$r'rr)#rdrrr.r0r/r rrSUFFIXfnamenum_input_shardspbarresultrrdonecontentexamples_per_job bytes_per_jobfeatures_per_jobshard_lengths_per_jobkwargs_per_jobnum_jobspooltotal_num_examplestotal_num_bytesrrshards_and_jobsrrrrrs#` @@@@rQrz$GeneratorBasedBuilder._prepare_splits2.2YFDYZZ+DH555$,@BGLL). 9  ')/*>?JJ(3J(9KK3KVKKkKK $*E22  ,1 >?YZZ 1$$)=\X\\*/\\\%*:X*E*E ]X]];K]]V`Ve]]CS]]],|7999)6z666    &,$$8     x1}}F(3JF - --GT-G.)&..**H $v0 !FX-M $v0 "Vh.N%)FX$5 !h 14111Cd8.22211-g 1!(  0 8 -f 5 0 8 .v 6 5f = =!KK00001111111111111111 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1&,,,,gM]ggg-,,>**  !122m,,#A&2D"//>",  7777888 !   U3Z         *3N*C*CO }otQS T T T T882G888O & 4 4 $ Hf LL g('8'899AA'f??[[ fb))    9  %!)DI    & %sI2GGGK AJ4( K 4J8 8K ;J8 <K  KKrrr.rrrvc #K|jd i|}|dkrtnt} |dk} g} d\} } d}d} | |jj|d|dd|d|j|j||jj | } tj }|D]A\}}||j |kr| \}}| | || |z } | |z } |dz }| |j|d|dd|d|j|j||jj | }|jj|jj|n|}||||dz }tj |t$jzkrtj }|d |fVd}C |d |fV|dz}| \}}| | || |z } | |z } n[#|d |fV|dz}| \}}| | || |z } | |z } wxYwnE#t($r8}t+|t,r|j|j}t1d |d}~wwxYw|d | | |j|| ffVdS) Nr5r'rrrr)rrr hash_saltcheck_duplicatesrembed_local_filesrF.An error occurred while generating the datasetTrO)rrrrrrrr\rrtime _num_bytesfinalizecloser _featuresencode_examplewriterPBAR_REFRESH_TIME_INTERVAL Exceptionrbr __context__rX)rdrrr.r/rrr generator writer_classrBr$r<r=rnum_examples_progress_updatewriter_timerrecordr+rIexamplerrs rQr,z+GeneratorBasedBuilder._prepare_split_single*s,D+99j99 (3y(@(@}}k '94 .2+O'($0 b!\+]]7x,=,=>>FFwSYP_P_``"&"9$/!5 $ 8"3F! - #,99KC%1f6G.6X6X28//2C2C/ i %,,\:::*l:*'94 A !-%+%5!&w88I8I!J!J!R!RSZ_e\k\k!l!l.2.E&0o-A,0H,D.?"""LP9K]Kidi0??GGGouGLL#...0A50y{{UV-N%NNN $ $e-IIIII784/92e%AAAAA%\ *0//*;*;' i $$\222"l2"9, e%AAAAA%\ *0//*;*;' i $$\222"l2"9, b b b!122 "q}7PM()YZZ`a a  b d/&BRT^`mnnnnnnns3AJEI,AJAJJ K!)3KK!c ~tj||fd|tjkp|tjki|dS)Nr)superrdr;rPrL)rdrr,prepare_splits_kwargs __class__s rQrdz+GeneratorBasedBuilder._download_and_preparepsb%%    "36F6S!S"@ $4$??  $      rPc6t|j|jSri)r,rrrs rQrz6GeneratorBasedBuilder._get_examples_iterable_for_splitys 79STTTrPr)rLrMrNrrrrr4r}r r~r r|rrr5rr tupler,rdr,r __classcell__rXs@rQrrps $$$D"&48 P*P*'P*#P* 3- P* !sCx1 P*P*P*P*dDoDoDo Do  Do  Do#DoDo %T5e#445 6DoDoDoDoL     UUScUUUUUUUUrPrceZdZdZejdZ ddedede e de e ee ffd Z d e d edede d e d eee ee e efff dZded efdZdS)ArrowBasedBuilderzaBase class for datasets with data generation based on Arrow loading functions (CSV/JSON/Parquet).c t)aDefault function generating examples for each `SplitGenerator`. This function preprocess the examples from the raw data to the preprocessed dataset files. This function is called once for each `SplitGenerator` defined in `_split_generators`. The examples yielded here will be written on disk. Args: **kwargs (additional keyword arguments): Arguments forwarded from the SplitGenerator.gen_kwargs Yields: key: `str` or `int`, a unique deterministic example identification key. * Unique: An error will be raised if two examples are yield with the same key. * Deterministic: When generating the dataset twice, the same example should have the same key. Good keys can be the image id, or line number if examples are extracted from a text file. The key will be hashed and sorted to shuffle examples deterministically, such as generating the dataset multiple times keep examples in the same order. example: `pyarrow.Table`, a feature table ready to be encoded and written to disk. rrs rQ_generate_tablesz"ArrowBasedBuilder._generate_tabless8"###rPr(Nrr.r0r/c X !t|p tj}tj }|rt jjn tj}j j j j |j }n|j }d}j d|j |d|} |j | |r|dkr{t|j} | dkr+|)t d|d|j dd}n6|4| |kr.t d|d| d |j d | d | }t%jt%j d |jd d|j d} ||d||dkrvd} |j} d}| 5jd(| |dD] \}}}|r|} | |! dddn #1swxYwY| Jdd| D\}}} }nfdt1t3|j|D}t5|}dg|z}dg|z}dg|z}dg|z dg|z}t7|5}| 5t9|j|D]5\}}}|r|\||<||<||< |<||< | |6 dddn #1swxYwYdddn #1swxYwYd|vsJd|dt; !t;|}t;|}|d}||j _||j _t d!d !dkr^dt@tBf !fd }dt1 D}tE||dd !d"|D|j _#n[d#\}}$%d$|d%%d&|d%%|d'j j&|j _&dSdS))NrrrrrrrrGrrrrFrrr)rr.r/rrrcg|]}|gSrOrOrs rQrz4ArrowBasedBuilder._prepare_split..rrPc$g|] \}}||d SrrOrs rQrz4ArrowBasedBuilder._prepare_split..rrPrrr r r shard_id_and_jobc"|\}}td||z}d|dd|dd|ddddSrr)rdrrrrrdrrs rQrz7ArrowBasedBuilder._prepare_split.._rename_shards#3 &"%nWfW&=">">"I MM'h+<+<==EEgRX__MM-O1I1IJJRRSZ_k\q\qrrrPc@g|]\}}t|D]}||fSrOrrs rQrz4ArrowBasedBuilder._prepare_split..sP"""&FJ %j 1 1""6"""""rPTrr cg|] }|D]}| SrOrOr#s rQrz4ArrowBasedBuilder._prepare_split..r&rPr'rrrr6rOr()"rdrr.r0r/r rrr.r/r0r1r2rrr3r4r5r6r7r8r9r:r;r<r=rrshard_ids_and_jobsrrrrrs"` @@@@rQrz ArrowBasedBuilder._prepare_splits2.2YFDYZZ+DH555$,@BGLL). 9  ')/*>?JJ(3J(9KK3KVKKkKK $*E22  ,1 >?YZZ 1$$)=\X\\*/\\\%*:X*E*E ]X]];K]]V`Ve]]CS]]],|7999)6z666    &,    x1}}F(3JF - --GT-G.)&..**H $v0 !FX-M $v0 "Vh.N%)FX$5 !h 14111Cd8.22211-g 1!(  0 8 -f 5 0 8 .v 6 5f = =!KK00001111111111111111 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1&,,,,gM]ggg-,,>**  !122m,,#A&2D"//>",  7777888 !   c          ""*3N*C*C"""  }&8$TV W W W W882G888O & 4 4 $ Hf LL g('8'899AA'f??[[ fb))    9  %!)DI    & %sI2F??GGK AJ2& K 2J6 6K 9J6 :K  K K rrrrvc #K|jd i|}|dkrtnt}|dk}g} d\} } d} d} ||jj|d| dd|d|j|jj|} tj }|D]\}}||j |kr| \}}| | || |z } | |z } | dz } ||j|d| dd|d|j|jj|}||| t!|z } tj |t"jzkrtj }|d | fVd}  |d | fV| dz}| \}}| | || |z } | |z } n[#|d | fV| dz}| \}}| | || |z } | |z } wxYwnE#t&$r8}t)|t*r|j|j}t/d |d}~wwxYw|d | | |j|| ffVdS) Nr5r'rrrr)rrrrrBrFrCTrO)r`rrrrrrrrrDrErFrGrrH write_tablerrrKrLrbrrMrX)rdrrr.r/rrNrOrBr$r<r=rrPrQrRrtabler+rIrrs rQr,z'ArrowBasedBuilder._prepare_split_single-sm*D)77J77 (3y(@(@}}k '94 .2+O'($+ b!\+]]7x,=,=>>FFwSYP_P_``"&"9 $ 8"3 F -  )99HAu%1f6G.6X6X28//2C2C/ i %,,\:::*l:*'94 A !-%+%5!&w88I8I!J!J!R!RSZ_e\k\k!l!l.2.E,0H,D.? """&&u---0CJJ>0y{{UV-N%NNN $ $e-IIIII784)9,e%AAAAA%\ *0//*;*;' i $$\222"l2"9, e%AAAAA%\ *0//*;*;' i $$\222"l2"9, b b b!122 "q}7PM()YZZ`a a  b d/&BRT^`mnnnnnnns3AI0 D0H=AI0AI,,I00 J2:3J--J2cRtt|j|jS)N)r)r,r.r`rrs rQrz2ArrowBasedBuilder._get_examples_iterable_for_splitgs- 243H I IRaRl    rPr)rLrMrNrrrr`r4r|r r~r rrrr r}rZr,r,rrOrPrQr^r^}s)kk$$$@#"&48 M*M*'M*M*3- M* !sCx1 M*M*M*M*^8o8o'*8o9<8oNQ8o[^8o %T5e#445 68o8o8o8ot  Sc      rPr^ceZdZdS)MissingBeamOptionsNrKrOrPrQrnrnmrRrPrnceZdZdZdddfd ZdZejdZfdZ dZ dd e e e effd Z dd e e d e ee efeffdZd ed efdZd efdZdZedZxZS)BeamBasedBuilderzBeam-based Builder.N) beam_runner beam_optionscd||_||_i|_tj|i|dSri) _beam_runner _beam_options _beam_writersrVr)rdrqrrargsrrXs rQrzBeamBasedBuilder.__init__ts=')$)&)))))rPci}tj|jj}d|vr |d|d<|S)Npipeline)rrrrrq)rdrqrsplit_generators_arg_namess rQrz.BeamBasedBuilder._make_split_generators_kwargszsO#%%,%6t7M%N%N%Y%^%^%`%`" 3 3 32Fz2R #J /&&rPc t)aBBuild the beam pipeline examples for each `SplitGenerator`. This function extracts examples from the raw data with parallel transforms in a Beam pipeline. It is called once for each `SplitGenerator` defined in `_split_generators`. The examples from the PCollection will be encoded and written to disk. Warning: When running in a distributed setup, make sure that the data which will be read (download_dir, manual_dir,...) and written (cache_dir) can be accessed by the workers jobs. The data should be located in a shared filesystem, like GCS. Args: pipeline ([`utils.beam_utils.BeamPipeline`]): Apache Beam pipeline. **kwargs (additional keyword arguments): Arguments forwarded from the SplitGenerator.gen_kwargs. Returns: `beam.PCollection`: Apache Beam PCollection containing the example to send to `self.info.features.encode_example(...)`. Example: ``` def _build_pcollection(pipeline, extracted_dir=None): return ( pipeline | beam.Create(gfile.io.listdir(extracted_dir)) | beam.Map(_process_file) ) ``` r)rdryrs rQ_build_pcollectionz#BeamBasedBuilder._build_pcollectionsJ"###rPc nddl}ddlmcm}|j}|j}|s-|s+d|jd|jjd}td|d|j t ddd i} d |vr3| d } | | d <| | d <d | d<td|p#|jjj| }|||} t'j|ft*j| d|| } | | } |jj}|jD]F\}}|j|}| | !|\}}||}||_"||_#tI|dr%tK|j&dkr |j&|_'|(dd}|jd|d|}|jd|d|}tS|j*stVj,j-n t\j-}||j/|}||j/|}|0||HdS)Nrrrz', beam_runner='DirectRunner')a*Trying to generate a dataset using Apache Beam, yet no Beam Runner or PipelineOptions() has been provided in `load_dataset` or in the builder arguments. For big datasets it has to run on large-scale data processing tools like Dataflow, Spark, etc. More information about Apache Beam runners at https://beam.apache.org/documentation/runners/capability-matrix/ If you really want to run it locally because you feel like the Dataset is small enough, you can use the local beam runner called `DirectRunner` (you may run out of memory). Example of usage: `rzf`writer_batch_size` is not supported for beam pipelines yet. Using the default chunk size for writing.pipeline_type_checkFr0direct_num_workers num_workersmulti_processingdirect_running_modezNUsing a DirectRunner with `num_proc` for multiprocessing it not supported yet.)runneroptions)r,ry)r_shard_lengthsrr.r(rz-00000-of-00001.r)1 apache_beamdatasets.utils.beam_utilsr beam_utilsrtrur\rrnrrrrrrpipeline_optionsPipelineOptionsfrom_dictionary BeamPipelinerVrdr;rKrunwait_until_finishmetricsrrfrvr MetricsFilterwith_namespacerFqueryr+rIrrrr$rr&rrrrrrr')rdrr,rWbeamrrqrr usage_examplerrrypipeline_resultsrr split_name beam_writerm_filterr+rIrr. src_fname dst_fnamer src_fpath dst_fpathrXs rQrdz&BeamBasedBuilder._download_and_preparesC""""666666666' )  < lTYllDK>**,,,"**,,Y% '+'9'?'?'A'A 3 3 #J |1133BBZBXXH&1&:&:7==;R;R&S&S #L)#J/J&2J ##,J {$455 3#k>X:Y:Y\]:]:]+6+E ((477 wOO #yTT:TT{TT #yEE:EE EE 0DTX0N0NbBGLLT]Tb %Id&6 BB %Id&6 BB  Y 2222# 3 3rPc0ddl}|jjj}t |jst jjn tj}| ||j tj 5}|j|dddn #1swxYwY|jjrc| ||j tj5}|j|ddddS#1swxYwYdSdS)Nr)rio filesystems FileSystemsr&rrrrrcreaterrrr _dump_infolicenseLICENSE_FILENAME _dump_license)rdrfsrfs rQrizBeamBasedBuilder._save_infosy"""" W ,(zMBeamBasedBuilder._prepare_split.._build_pcollection..s!6!9nn]cde]fNgNgBhrP)r|rMapwrite_from_pcollection)rypcoll_examplesrrrIrdrs rQr|z;BeamBasedBuilder._prepare_split.._build_pcollections[5T4X\\A[\\N h$((3h3h3h3h*i*ii iN55nEE ErP)rrrr\r&rrrrrrrrrrvrI ptransform_fn)rdrryr.r/rr/rrr|rrrrIs`` @@@rQrzBeamBasedBuilder._prepare_splits6 #"""  %%n  %/4 999z99K99(.(sR    J(M(Me(T(T[_[dlqlvwww   rPrr) _request_info_from_hf_gcsrrfrKeyErrorrcrrbrr)rdrrs` rQrz%BeamBasedBuilder.as_streaming_dataset#s &&(((    )0022     \ \#E? \ \ \ !Zu!Z!Z$x..!Z!Z[[[ \ h % % 5*844Hs A -A9c0t|jd|iS)Nr)r,_generate_examples_from_hf_gcsrs rQrz1BeamBasedBuilder._get_examples_iterable_for_split5s CguEUVVVrPc# Kjr2tj fdt D}njdjdjdg}d}|D]}t |d5}t j|5}|D]$}| D] }||fV|dz }% dddn #1swxYwYdddn #1swxYwYdS)Nc Tg|]$}jdjdjd|dddd %S)rrrz-of-.arrow)_remote_cache_dir_from_hf_gcsr\)rkrrrdrs rQrzCBeamBasedBuilder._generate_examples_from_hf_gcs..;sa$$$5yy yyEJyyQYyyycmyyyy$$$rPrrrrrbr) r$rrrr\r$paipc open_stream to_pylist) rdrremote_prepared_urlsrremote_prepared_urlrr} record_batchrSrs `` @rQrz/BeamBasedBuilder._generate_examples_from_hf_gcs8s   lU011J$$$$$$ %j 1 1$$$ (,'I$j$jDI$j$jX]Xb$j$j$j#k #7 % % *D11 %QV''**%f(.%% &2&<&<&>&>%%F"%v+---1HCC%%%%%%%%%%%%%%%%% % % % % % % % % % % % % % % % % %s60 C(C9 CC C C CC #C cVddlm}|jdtj} ||5}ddl}||}dddn #1swxYwYn"#t$r}t|dd}~wwxYw|j tj |dS)Nr)r$rr) #download.streaming_download_managerr$rrrjsonloadrrrrr) from_dict)rdr$remote_dataset_inforrrerrs rQrz*BeamBasedBuilder._request_info_from_hf_gcsJs>>>>>>!%!CddfFbdd 8*++ %q  !  % % % % % % % % % % % % % % %! 8 8 8(--4 7 8 .u5566666s: AA AAAAA A:%A55A:c|d}tdz|tjdzS)NF)r r)rrrrr)rdr|s rQrz.BeamBasedBuilder._remote_cache_dir_from_hf_gcsWs< 33e3DD$'8'@'@'M'MMMrP)r(Nri)rLrMrNrrrrrr|rdrir r r|r~rrr-rr5r,rrrrrr[r\s@rQrprpqs*.T******* ''' $$$$$$LG3G3G3G3G3R + + +ko":":NVW\]`be]eWfNg":":":":L $} tC()?: ;$WiWDTWWWW%I%%%%$ 7 7 7NNXNNNNNrPrp)rrrVrrrrrBrurDrr dataclassesr functoolsrpathlibrtypingrrrr r r rpyarrowr multiprocessr tqdm.contrib.concurrentr r6rr arrow_datasetr arrow_readerrrrrr arrow_writerrrrrr_rr dataset_dictrrdownload.download_configrdownload.download_managerr r!download.mock_download_managerr"rr#r$rr%rr&r'rr(rr)r*r+iterable_datasetr,r-r.keyhashr/namingr0r1rfr2r3r4r5 streamingr6r7utils.file_utilsr8r9utils.filelockr:utils.info_utilsr;r<r=r>utils.py_utilsr?r@rArBrCrDrErFutils.shardingrGrH get_loggerrLrrcrJrLrTrVrXrZrrr^rnrprOrPrQrs !     !!!!!!BBBBBBBBBBBBBBBB ......""""""WVVVVVVVVVVV88888888::::::::444444DDDDDDDD??????PPPPPPPP BBBBBBBBBBgggggggggg((((((NNNNNNNN????????????;;;;;;88888888$$$$$$gggggggggggg                    ONNNNNNN  H % %                     +        .    ^^^^^^^ ^Bd$d$d$d$d$d$d$d$N%JUJUJUJUJUNJUJUJUZm m m m m m m m `        iNiNiNiNiN~iNiNiNiNiNrP