+gdVdZddlZddlZddlZddlZddlZddlZddlZddlmZddl m Z ddl m Z ddl mZmZmZmZmZmZmZmZddlmZdd lmZdd lmZmZmZmZmZdd l m!Z!dd l"m#Z#m$Z$m%Z%dd l&m'Z'm(Z(m)Z)ddl*m+Z+e#e,Z-gdZ.e/0dde/0dde/0dde/0dde/0dde/0dde/0dde/0ddiZ1dd iZ2e3d!e e1e2DZ4Gd"d#ej5Z6Gd$d%eZ7d&e8d'e8fd(Z9d'ee8fd)Z:d&e8d'ee8fd*Z;Gd+d,eZ<Gd-d.e<Z=Gd/d0e<Z>Gd1d2Z?dS)3zDownload manager interface.N)datetime)partial)chain)CallableDict GeneratorIterableListOptionalTupleUnion)config)DeprecatedEnum) cached_pathget_from_cachehash_url_to_filenameis_relative_pathurl_or_path_join)get_size_checksum_dict) get_loggeris_progress_bar_enabledtqdm)NestedDataStructure map_nestedsize_str)DownloadConfig) txtcsvjsonjsonltsvconllconlluorigparquetpklpicklerelxml504B0304zip504B0506504B0708425A68bz21F8Bgzip FD377A585A00xz04224D18lz428B52FFDzstdsRar!rarc#4K|]}t|VdSN)len).0 magic_numbers Blib/python3.11/site-packages/datasets/download/download_manager.py rAFs> ceZdZdZdZdZdZdS) DownloadModea)`Enum` for how to treat pre-existing downloads and data. The default mode is `REUSE_DATASET_IF_EXISTS`, which will reuse both raw downloads and the prepared dataset if they exist. The generations modes: | | Downloads | Dataset | |-------------------------------------|-----------|---------| | `REUSE_DATASET_IF_EXISTS` (default) | Reuse | Reuse | | `REUSE_CACHE_IF_EXISTS` | Reuse | Fresh | | `FORCE_REDOWNLOAD` | Fresh | Fresh | reuse_dataset_if_existsreuse_cache_if_existsforce_redownloadN)__name__ __module__ __qualname____doc__REUSE_DATASET_IF_EXISTSREUSE_CACHE_IF_EXISTSFORCE_REDOWNLOADrBr@rDrDLs.  83)rBrDc0eZdZdZdZdZedZdS) GenerateModerErFrGcdS)NzUse 'DownloadMode' instead.rOselfs r@ help_messagezGenerateMode.help_messagefs,,rBN)rHrIrJrLrMrNpropertyrUrOrBr@rQrQas?73) --X---rBrQpathreturnc||dd}dD]}||d}|S)N.z?-_r)split)rW extensionsymbs r@_get_path_extensionr_ksE 3#I--OOD))!, rBc |dn#ttjf$rYdSwxYw|t }|dt t D]q}t|dt |z }||cSt|dt |z }|td|drdS)zQread the magic number from a file-like object and return the compression protocolrNzCompression protocol 'z' not implemented.) seekAttributeErrorioUnsupportedOperationreadMAGIC_NUMBER_MAX_LENGTHrange$MAGIC_NUMBER_TO_COMPRESSION_PROTOCOLget0MAGIC_NUMBER_TO_UNSUPPORTED_COMPRESSION_PROTOCOLNotImplementedError)fr?i compressions r@*_get_extraction_protocol_with_magic_numberrous  q B3 4tt66122LFF1III * + +``:>>|LiNehiNiLi?jkk  "   FJJ49< <<<<<<<<<<A--A14A1c$||j|Sr<)r)rrs r@from_bufzArchiveIterable.from_bufss3)7333rBc$||j|Sr<)r)rurlpath_or_bufs r@ from_pathzArchiveIterable.from_pathss3&777rB)rXr)rHrIrJrK staticmethodrr classmethodrr rrtrrrrOrBr@rrsSS\  & &\ &(itT0A&B((([(,c,itT8I.J,,,[,444[4888[888rBrcxeZdZdZedeeeefdeeddffdZ eddZ dS) FilesIterablez8An iterable of paths from a list of directories or filesurlpathsrXNc#fKt|ts|g}|D]}tj|r:tj|drdS|V\tj|D]\}}}td|D|dd<tj|drWt|D]:}|drtj ||V;dS)Nrc<g|]}|d|S)r)r)r>dirnames r@ z2FilesIterable._iter_from_paths..s-)q)q)qgQXQcQcdoQpQp)q')q)q)qrB) isinstancelistrrWisfilerrwalksortedjoin)rrrdirpathdirnames filenamesrs r@_iter_from_pathszFilesIterable._iter_from_pathssN(D)) " zH > >Gw~~g&& >7##G,,77 DDFF 46GG4D4D > >0GXy#))q)q)q)q)q"r"rHQQQKw''00;;KHH! $*9$5$5>>#..{;;%$ gll7H====== > > > >rBc$||j|Sr<)r)rrs r@ from_pathszFilesIterable.from_pathsss3'222rB)rXr) rHrIrJrKrr rtr rrrrOrBr@rrsBB>c49n(=>)CQUW[OB\>>>[>.333[333rBrc FeZdZdZ ddeedeedeedeefdZed Z ed Z e d Z d e d e fdZdZdZdededefdZdeeejffdZdeeeeffdZddZdZdZdZdZdS) DownloadManagerFNT dataset_namedata_dirdownload_config base_pathc||_||_|ptjd|_i|_||_|p t|_ i|_ i|_ dS)a4Download manager constructor. Args: data_dir: can be used to specify a manual directory to get the files from. dataset_name (`str`): name of dataset this instance will be used for. If provided, downloads will contain which datasets they were used for. download_config (`DownloadConfig`): to specify the cache directory and other download options base_path (`str`): base path that is used when relative paths are used to download files. This can be a remote url. record_checksums (`bool`, defaults to `True`): Whether to record the checksums of the downloaded files. If None, the value is inferred from the builder. rZN) _dataset_name _data_dirrrWabspath _base_path_recorded_sizes_checksumsrecord_checksumsrrdownloaded_pathsextracted_paths)rTrrrrrs r@rzDownloadManager.__init__sf2*!#;rws';';Z\& 0.B.2B2B "!rBc|jSr<)rrSs r@ manual_dirzDownloadManager.manual_dir$s ~rBcbtd|jDS)z+Returns the total size of downloaded files.c3&K|] }|dV dS) num_bytesNrO)r>checksums_dicts r@rAz2DownloadManager.downloaded_size..+s'mm>>+.mmmmmmrB)sumrvaluesrSs r@downloaded_sizezDownloadManager.downloaded_size(s0mmTEcEjEjElElmmmmmmrBcddlm|jdt dfdt fd|t }|S) aShip the files using Beam FileSystems to the pipeline temp dir. Args: downloaded_path_or_paths (`str` or `list[str]` or `dict[str, str]`): Nested structure containing the downloaded path(s). pipeline ([`utils.beam_utils.BeamPipeline`]): Apache Beam Pipeline. Returns: `str` or `list[str]` or `dict[str, str]` r)upload_local_to_remote temp_locationNzFYou need to specify 'temp_location' in PipelineOptions to upload filesc 6tjtjtj|}td|dttj |d|d|||S)Nz Uploading z (z) to rZ) posixpathrrDOWNLOADED_DATASETS_DIRrrWrloggerinforgetsize)local_file_pathremote_file_path remote_dirrs r@uploadz8DownloadManager.ship_files_with_pipeline..uploadAs(~F:BGz:DownloadManager.ship_files_with_pipeline..LsFF?$;$;rB disable_tqdm)utils.beam_utilsr_optionsget_all_optionsri ValueErrorrr)downloaded_path_or_pathspipelineuploaded_path_or_pathsrrrs @@@r@ship_files_with_pipelinez(DownloadManager.ship_files_with_pipeline-s >=====&6688<<_MM  eff f $ $ $ $ $ $", ; ; ; ; $4666" " "  &%rB url_or_urlsrc *d}ttt|||dt D]0\}}t ||j|jt|<1dS)z)Record size/checksum of downloaded files.zComputing checksums)delaydescdisable)record_checksumN) rrr-flattenrrrrrt)rTrrrurlrWs r@_record_sizes_checksumsz'DownloadManager._record_sizes_checksumsRs [((**,D,L,L,N,NOO P P&/111     IC8Nd&;888D *3s88 4 4  rBc. |jjp tj |jj} fd}t ||t }t|}t|}t| | D]]\}} t| dd|d}n#t$rd}YnwxYw|r |jj r |||t| dd|^| |||jS)a Download given urls(s) by calling `custom_download`. Args: url_or_urls (`str` or `list` or `dict`): URL or `list` or `dict` of URLs to download and extract. Each URL is a `str`. custom_download (`Callable[src_url, dst_path]`): The source URL and destination path. For example `tf.io.gfile.copy`, that lets you download from Google storage. Returns: downloaded_path(s): `str`, The downloaded paths matching the given input `url_or_urls`. Example: ```py >>> downloaded_files = dl_manager.download_custom('s3://my-bucket/data.zip', custom_download_for_my_private_bucket) ``` c^tjt|Sr<)rrWrr)r cache_dirs r@url_to_downloaded_pathz?DownloadManager.download_custom..url_to_downloaded_pathxs"7<< +?+D+DEE ErBrTF)rlocal_files_onlyuse_etag max_retries)rrrDOWNLOADED_DATASETS_PATHrrrrr-rrFileNotFoundErrorforce_downloadrdata) rTrcustom_downloadrrrrrWcachedrs @r@download_customzDownloadManager.download_custom`s*(2Uf6U *6  F F F F F$. "KBYB[B[>[$ $ $  *+66 #67O#P#P [00224L4T4T4V4VWW  IC 9teal$     T1@ T***9teal $$[2JKKK',,s"B99 CCc p|j}d|_|jd|_t |j|}t j}t||d|j t d}t j|z }t d| d zd t|}t|}|jt#t%||t j}|||t j|z }t d | d zd |jS) ayDownload given URL(s). By default, only one process is used for download. Pass customized `download_config.num_proc` to change this behavior. Args: url_or_urls (`str` or `list` or `dict`): URL or `list` or `dict` of URLs to download. Each URL is a `str`. Returns: `str` or `list` or `dict`: The downloaded paths matching the given input `url_or_urls`. Example: ```py >>> downloaded_files = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz') ``` FNDownloading datarTzDownloading data files) map_tuplenum_procrrzDownloading took <z minzChecksum Computation took )rcopyextract_compressed_file download_descr _downloadrnowrr rrr total_secondsrrupdatedictr-rrr)rTrr download_func start_timerdurations r@downloadzDownloadManager.downloads&.335527/  ( 0,>O )PPP \^^ #-  $-4666) $ $ $  <>>J. L(>(>(@(@B(FLLLMMM)+66 #67O#P#P  $$T#k.A.A.C.CE]EeEeEgEg*h*h%i%ijjj\^^  $$[2JKKK<>>J. U1G1G1I1IR1OUUUVVV',,rBurl_or_filenamerXct|}t|rt|j|}t ||S)Nr )rtrrrr)rTrrs r@rzDownloadManager._downloadsDo.. O , , Q.tPPO?OLLLLrB path_or_bufct|drt|St|S)aKIterate over files within an archive. Args: path_or_buf (`str` or `io.BufferedReader`): Archive path or archive binary file object. Yields: `tuple[str, io.BufferedReader]`: 2-tuple (path_within_archive, file_object). File object is opened in binary mode. Example: ```py >>> archive = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz') >>> files = dl_manager.iter_archive(archive) ``` re)hasattrrrr)rTrs r@ iter_archivezDownloadManager.iter_archives>( ; ' ' :"++K88 8",,[99 9rBpathsc6t|S)aIterate over file paths. Args: paths (`str` or `list` of `str`): Root paths. Yields: `str`: File path. Example: ```py >>> files = dl_manager.download_and_extract('https://huggingface.co/datasets/beans/resolve/main/data/train.zip') >>> files = dl_manager.iter_files(files) ``` )rr)rTr!s r@ iter_fileszDownloadManager.iter_filess"''...rB deprecatedc |dkrtjdt|j}d|_|jd|_ttt|||j t d}t|}t|}|j tt!|||jS) akExtract given path(s). Args: path_or_paths (path or `list` or `dict`): Path of file to extract. Each path is a `str`. num_proc (`int`): Use multi-processing if `num_proc` > 1 and the length of `path_or_paths` is larger than `num_proc`. Pass `DownloadConfig(num_proc=)` to the initializer instead. Returns: extracted_path(s): `str`, The extracted paths matching the given input path_or_paths. Example: ```py >>> downloaded_files = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz') >>> extracted_files = dl_manager.extract(downloaded_files) ``` r$z'num_proc' was deprecated in version 2.6.2 and will be removed in 3.0.0. Pass `DownloadConfig(num_proc=)` to the initializer instead.TNr r zExtracting data files)r rr)warningswarn FutureWarningrrrrrrrr rrrrrr-rr)rT path_or_pathsr rrs r@extractzDownloadManager.extracts6 | # # Mb   .335526/  ( 0,>O )$ K A A A $-4666(    ,M:: -o>> ##D]-B-B-D-DoF]F]F_F_)`)`$a$abbb##rBcR|||S)aDownload and extract given `url_or_urls`. Is roughly equivalent to: ``` extracted_paths = dl_manager.extract(dl_manager.download(url_or_urls)) ``` Args: url_or_urls (`str` or `list` or `dict`): URL or `list` or `dict` of URLs to download and extract. Each URL is a `str`. Returns: extracted_path(s): `str`, extracted paths of given URL(s). )r*r)rTrs r@download_and_extractz$DownloadManager.download_and_extract$s" ||DMM+66777rBc4|jSr<)rrrSs r@get_recorded_sizes_checksumsz,DownloadManager.get_recorded_sizes_checksums6s-22444rBcvt|jt|jz }t |jD]D\}}||vr;t j|rt j ||j|=EdSr<) setrrrritemsrrWrremove)rTpaths_to_deletekeyrWs r@delete_extracted_filesz&DownloadManager.delete_extracted_files9sd299;;<>$+?+?& $(- . .rBcJ|jjr|dSdSr<)rdelete_extractedr5rSs r@manage_extracted_filesz&DownloadManager.manage_extracted_files@s2   0 *  ' ' ) ) ) ) ) * *rB)NNNNT)r$)rHrIrJ is_streamingr rtrrrVrrrrrrrrrr rcBufferedReaderr r r#r*r,r.r5r8rOrBr@rrsL'+"&48#' !"!"sm!"3-!"".1 !" C= !"!"!"!"FXnnXn"&"&\"&H 3F bu    .-.-.-`.-.-.-`MM~MRUMMMM:c23D.D(E::::2/c49n 5////&/$/$/$/$b888$555...*****rBr)@rKenumrcrrrr&rr functoolsr itertoolsrtypingrrrr r r r r rutils.deprecation_utilsrutils.file_utilsrrrrrutils.info_utilsr utils.loggingrrrutils.py_utilsrrrrrrHrrubytesfromhexrhrjmaxrfEnumrDrQrtr_rorxrzrrrrOrBr@rIs "! TTTTTTTTTTTTTTTTTTTT444444tttttttttttttt555555EEEEEEEEEEFFFFFFFFFF++++++ H    MM*u MM*u MM*u MM(U MM&6 MM.!!4 MM*u MM*v ($ U40#BDtuu *****49****----->---cc`Xc]````$ =3 =8C= = = = = = = = = =X = = =:8:8:8:8:8,:8:8:8z33333*333@D*D*D*D*D*D*D*D*D*D*rB