
    +gd\             +       6   d Z ddlZddlZddlZddlZddlZddlZddlZddlZddl	Z	ddl
mZ ddlmZ ddlmZ ddlmZmZmZmZmZmZmZmZ ddlZddlZddlmZ dd	lmZ dd
lm Z  ddl!m"Z" ddl#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z, ddl-m.Z.m/Z/ ddl0m1Z1 ddl2m3Z3 ddl4m5Z5m6Z6m7Z7 ddl8m9Z9 ddl:m;Z;m<Z< ddl=m>Z>m?Z? ddl@mAZA ddlBmCZC ddlDmEZEmFZFmGZGmHZH ddlImJZJ ddlKmLZL ddlMmNZN ddlOmPZPmQZQmRZRmSZSmTZTmUZUmVZVmWZWmXZX ddlYmZZZ ddl[m\Z\ ddl]m^Z^m_Z_ ddl`maZa ddlbmcZc dd ldmeZe dd!lfmgZg  eaeh          Zi ej eEjk                              d"gz   Zlejm        dfd#end$eeeenf                  fd%Zodwd'eeee"         eeC         f                  fd(Zpd)een         d'enfd*Zqd#end+enfd,Zrd#end-end.eenenenenf         d/ee1         d'eeenenf                  f
d0Zsd#end1end2end3end4eeenenf                  d5eeenenf                  d6eee3enf                  d'enfd7Ztd8end4eeenenf                  d5eeenenf                  d9end:end#end6e3d'eenenf         fd;Zu	 dxd<e&d=eeevenf                  d'eeenenf                  fd>Zwd<e&d=eeevenf                  d'eeenenf                  fd?Zxe G d@ dA                      Zye G dB dC                      Zz G dD dE          Z{ G dF dG          Z| G dH dIe|          Z} G dJ dKe|          Z~ G dL dMe{          Z G dN dOe{          Z G dP dQe{          Z G dR dSe{          Z G dT dUe{          Z G dV dWe{          Z G dX dYe|          Z	 	 	 	 	 	 dydZend[eeenegf                  d/ee1         d6eee3enf                  d9een         d\een         d]eeeeene%f                  d'eyfd^Z eNd_          	 	 	 	 dzdZend[eeenegf                  d/ee1         d6eee3enf                  d9een         d'ezfd`            Z eNda          	 	 	 	 	 	 	 	 	 d{dZendceen         ddedeedfeen         dgeen         dhevd/ee1         d6eee3enf                  d[eeenegf                  d'eCfdi            Z	 	 	 	 	 	 	 	 	 	 d|dZend#een         d\een         d]eeeneen         eeneeneen         f         f         f                  dfeen         djee9         d/ee1         d6eee3enf                  d[eeenegf                  d=eeevenf                  dkee         d'e"fdlZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d}dZend#een         d\een         d]eeeneen         eeneeneen         f         f         f                  dneeeneJf                  dfeen         djee9         d/ee1         d6eee3enf                  doeee^enf                  dheev         dpevd[eeenegf                  d=eeevenf                  dqeeeneLf                  drevdsee         dkee         d'ee.e e/eAf         f&dtZ	 d~duendheev         dkee         d'ee e.f         fdvZdS )zAccess datasets.    N)Counter)	dataclass)Path)DictListMappingOptionalSequenceTupleTypeUnion)HfApi   )config)Dataset)DatasetBuilder)	DEFAULT_PATTERNS_ALLDataFilesDictDataFilesListEmptyDatasetError'get_data_patterns_in_dataset_repositoryget_data_patterns_locally+get_metadata_patterns_in_dataset_repositoryget_metadata_patterns_locallysanitize_patterns)DatasetDictIterableDatasetDict)DownloadConfig)DownloadMode)StreamingDownloadManagerxglobxjoin)Features)extract_path_from_uriis_remote_filesystem)DatasetInfoDatasetInfosDict)IterableDataset)Metric)_EXTENSION_TO_MODULE_MODULE_SUPPORTS_METADATA_PACKAGED_DATASETS_MODULES_hash_python_lines)Split)TaskTemplate)
deprecated)	OfflineModeIsEnabled!_raise_if_offline_mode_is_enabledcached_path
head_hf_s3hf_github_urlinit_hf_modulesis_relative_pathrelative_to_absolute_pathurl_or_path_join)FileLock)
hf_hub_url)VerificationModeis_small_dataset)
get_logger)DatasetMetadata)get_imports)Versionzipnamehf_modules_cachec                    t          |          }t          j                            ||           }t          j        |d           t          j                            t          j                            |d                    sGt          t          j                            |d          d          5  	 ddd           n# 1 swxY w Y   |S )a^  
    Create a module with name `name` in which you can add dynamic modules
    such as metrics or datasets. The module can be imported using its name.
    The module is created in the HF_MODULE_CACHE directory by default (~/.cache/huggingface/modules) but it can
    be overridden by specifying a path to another directory in `hf_modules_cache`.
    Texist_ok__init__.pywN)r6   ospathjoinmakedirsexistsopen)rC   rD   dynamic_modules_paths      -lib/python3.11/site-packages/datasets/load.pyinit_dynamic_modulesrR   \   s     ''7887<<(8$??K$t44447>>"',,';]KKLL "',,3]CCSII 	 		 	 	 	 	 	 	 	 	 	 	 	 	 	 	s   1B??CCTreturnc                 D   t          j        |           }|rt          }nt          }d}|j                                        D ]^\  }}t          j        |          rEt          ||          r5t          j	        |          r>|}t          j
        |          }|||k    r n_|S )zImport a module at module_path and return its main class:
    - a DatasetBuilder if dataset is True
    - a Metric if dataset is False
    N)	importlibimport_moduler   r)   __dict__itemsinspectisclass
issubclass
isabstract	getmodule)module_pathdatasetmodulemain_cls_typemodule_main_clsrC   obj
obj_modules           rQ   import_main_classre   n   s    
 $[11F & O_**,,  	c?3 	JsM$B$B 	!#&& !O *3//J%&J*>*>    
file_pathsc                    g }| D ]y}t           j                            |          rC|                    t	          t          |                              d                               d|                    |           zg }|D ]R}t          |d          5 }|                    |	                                           ddd           n# 1 swxY w Y   St          |          S )zt
    Convert a list of scripts or text files provided in file_paths into a hashed filename in a repeatable way.
    z
*.[pP][yY]utf-8encodingN)rJ   rK   isdirextendlistr   rglobappendrO   	readlinesr-   )rg   to_use_files	file_pathlinesfs        rQ   files_to_hashrv      s   
 ,.L + +	7==## 	+T)__%:%:<%H%H I IJJJJ	**** E! ( (	)g... 	(!LL'''	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	(e$$$s   (C

C	C	resource_typec                     t           j        s9t           j        r/	 t          | | dz   |dk               dS # t          $ r Y dS w xY wdS dS )z1Update the download count of a dataset or metric..pyr_   )filenamer_   N)r   HF_DATASETS_OFFLINEHF_UPDATE_DOWNLOAD_COUNTSr4   	Exception)rC   rw   s     rQ   increase_load_countr~      sv    % &*J 	tdUl]i=WYYYYYY 	 	 	DD	   s   5 
AA	base_pathimportsdownload_configc                    g }g }|                                 }|j        d|_        |D ]\  }}}}	|dk    r|                    ||f           %|| k    rt          d|  d| d| d| d	          |d	k    rt	          ||d
z             }
n|dk    r|}
nt          d          t          |
|          }|	 t          j                            ||	          }|                    ||f           i }|D ]9\  }}	 t          j
        |          }# t          $ r ||vs||k    r|||<   Y 6w xY w|rt          |          dk    rdnd}t          |          dk    rdnd}d|                                v rd|d<   t          d|  d| dd                    |           d| dd                    |                                           d          |S )a  
    Download additional module for a module <name>.py at URL (or local path) <base_path>/<name>.py
    The imports must have been parsed first using ``get_imports``.

    If some modules need to be installed with pip, an error is raised showing how to install them.
    This function return the list of downloaded modules as tuples (import_name, module_file_path).

    The downloaded modules can then be moved into an importable directory with ``_copy_script_and_other_resources_in_importable_dir``.
    NzDownloading extra moduleslibraryzError in the z script, importing relative z module but z: is the name of the script. Please change relative import zl to another name and add a '# From: URL_OR_PATH' comment pointing to the original relative import file path.internalry   externalzWrong import_typer   r   dependencies
dependencythemitsklearnzscikit-learnzTo be able to use z$, you need to install the following : z, z.
Please install z using 'pip install  z' for instance.)copydownload_descrp   
ValueErrorr9   r3   rJ   rK   rL   rU   rV   ImportErrorlenkeysvalues)rC   r   r   r   local_importslibrary_importsimport_typeimport_nameimport_pathsub_directoryurl_or_filenamelocal_import_pathneeds_to_be_installedlibrary_import_namelibrary_import_pathlib_dependencies_str	_them_strs                     rQ   _download_additional_modulesr      s    MO%**,,O$,(C%@G ? ?<[+})##""K#=>>>$O O O+ O O"O O1<O O O   *$$.y+:MNNOOJ&&)OO0111'+
 
 
 $ "-> N Nk+<=>>>> 4C Q Q00	Q)*=>>CC 	Q 	Q 	Q"*???CVZmCmCm=P%&9:	Q  	
.12G.H.H1.L.LNNR^!"7881<<FF$	-224444/=!),I I IK\ I Iyy.//I ICLI Ixx-446677I I I
 
 	

 s   7DD('D(importable_directory_pathsubdirectory_nameoriginal_local_pathr   additional_filesdownload_modec                    t           j                            ||          }t           j                            || dz             }|dz   }	t          |	          5  |t          j        k    r3t           j                            |          rt          j        |           t          j	        |d           t           j                            |d          }
t           j                            |
          s)t          |
d          5  	 ddd           n# 1 swxY w Y   t          j	        |d           t           j                            |d          }
t           j                            |
          s)t          |
d          5  	 ddd           n# 1 swxY w Y   t           j                            |          st          j        ||           t           j                            |          d         d	z   }t           j                            |          sD||d
}t          |dd          5 }t          j        ||           ddd           n# 1 swxY w Y   |D ]\  }}t           j                            |          rXt           j                            ||dz             }t           j                            |          st          j        ||           }t           j                            |          rUt           j                            ||          }t           j                            |          st          j        ||           t%          d|           |D ]n\  }}t           j                            ||          }t           j                            |          rt'          j        ||          st          j        ||           o|cddd           S # 1 swxY w Y   dS )a  Copy a script and its required imports to an importable directory

    Args:
        name (str): name of the resource to load
        importable_directory_path (str): path to the loadable folder in the dynamic modules directory
        subdirectory_name (str): name of the subdirectory in importable_directory_path in which to place the script
        original_local_path (str): local path to the resource script
        local_imports (List[Tuple[str, str]]): list of (destination_filename, import_file_to_copy)
        additional_files (List[Tuple[str, str]]): list of (destination_filename, additional_file_to_copy)
        download_mode (Optional[Union[DownloadMode, str]]): download mode

    Return:
        importable_local_file: path to an importable module with importlib.import_module
    ry   z.lockTrF   rH   rI   Nr   z.json)zoriginal file pathzlocal file pathri   rj   zError with local import at )rJ   rK   rL   r:   r   FORCE_REDOWNLOADrN   shutilrmtreerM   rO   copyfilesplitextjsondumpisfilerl   copytreer   filecmpcmp)rC   r   r   r   r   r   r   importable_subdirectoryimportable_local_file	lock_pathinit_file_path	meta_pathmeta	meta_filer   r   full_path_local_import	file_nameoriginal_pathdestination_additional_paths                       rQ   2_copy_script_and_other_resources_in_importable_dirr      s   6 !gll+DFWXXGLL)@$,OO)G3I	)		 4% 4%L999bgnnMf>g>g9M3444
-==== &?OOw~~n-- 	nc**                 	+d;;;;&=}MMw~~n-- 	nc**                 w~~344 	HO/1FGGG G$$%:;;A>H	w~~i(( 	+*=RghhDiw777 +9	$	***+ + + + + + + + + + + + + + + )6 
	O 
	O$Kw~~k** 	O)+6M{]bOb)c)c&w~~&<== IOK1GHHH{++ O)+6M{)[)[&w~~&<== IOK1GHHH!"M"M"MNNN )9 	L 	L$I}*,',,7NPY*Z*Z'7>>"=>> Lgk:G G L /JKKK$i4% 4% 4% 4% 4% 4% 4% 4% 4% 4% 4% 4% 4% 4% 4% 4% 4% 4%s   B)ODOD	OD	A(O?FOF	OF	BO*I OI	OI	E:OO"O
local_pathrP   module_namespacec           	         t           j                            |||                    dd                    }t	          |                              dd           t	          |          j        dz                      d           t          | gd |D             z             }t          |
                    d          d         ||| |||	          }	t                              d
|	            d                    t           j                            |          ||                    dd          ||
                    d          d         g          }
|
|fS )N/--T)parentsrG   rH   rF   c                     g | ]
}|d          S )r    ).0locs     rQ   
<listcomp>z+_create_importable_file.<locals>.<listcomp>H  s    (I(I(ICQ(I(I(Irf   )rC   r   r   r   r   r   r   z#Created importable dataset file at .)rJ   rK   rL   replacer   mkdirparenttouchrv   r   splitloggerdebugbasename)r   r   r   rP   r   rC   r   r   hashr   r^   s              rQ   _create_importable_filer   <  s\    !#-ACSUYUaUabegkUlUl m m	"##))$)FFF	
#$$+m;BBDBQQQ*(I(I=(I(I(IIJJDNZZ__R ";&#)#   LLN7LNNOOO((			.	/	/1A4<<PSUYCZCZ\`bfblblmpbqbqrtbuv K rf   data_files_listuse_auth_tokenc                     t          d | dt          j                 D                       }|rK|                                D ]6\  }}|t          v rt          |         c S |dk    rt          | |          c S 7di fS )a  Infer module (and builder kwargs) from list of data files.

    Args:
        data_files_list (DataFilesList): List of data files.
        use_auth_token (bool or str, optional): Whether to use token or token to authenticate on the Hugging Face Hub
            for private remote files.

    Returns:
        tuple[str, str]: Tuple with
            - inferred module name
            - builder kwargs
    c              3   X   K   | ]%}t          |          j        D ]}|d d         V  &dS r   Nr   suffixesr   filepathsuffixs      rQ   	<genexpr>z.infer_module_for_data_files.<locals>.<genexpr>h  s^       ! !8nn-! !  	qrr
! ! ! ! ! ! !rf   NrB   r   )r   r   *DATA_FILES_MAX_NUMBER_FOR_MODULE_INFERENCEmost_commonr*   'infer_module_for_data_files_in_archives)r   r   extensions_counterext_s        rQ   infer_module_for_data_filesr   Y  s     ! ! !'([&*[([\! ! !  
  o(4466 	o 	oFC***+C0000>_mnnnnnn 8Orf   c                    g }d}| D ]}t          |                              d          rw|dz  }|t          j        k    r nat	          t                                          |          d          }|d t          |d|          dt          j                 D             z  }t          d	 |D                       }|r7|
                    d          d         d         }|t          v rt          |         S di fS )
a  Infer module (and builder kwargs) from list of archive data files.

    Args:
        data_files_list (DataFilesList): List of data files.
        use_auth_token (bool or str, optional): Whether to use token or token to authenticate on the Hugging Face Hub
            for private remote files.

    Returns:
        tuple[str, str]: Tuple with
            - inferred module name
            - builder kwargs
    r   z.zipr   z**c                 D    g | ]}|                     d           d         S )z::r   r   )r   ru   s     rQ   r   z;infer_module_for_data_files_in_archives.<locals>.<listcomp>  s7        a   rf   T)	recursiver   Nc              3   X   K   | ]%}t          |          j        D ]}|d d         V  &dS r   r   r   s      rQ   r   z:infer_module_for_data_files_in_archives.<locals>.<genexpr>  sD       q qY]^fYgYgYp q qv q q q q q q qrf   )strendswithr   2GLOBBED_DATA_FILES_MAX_NUMBER_FOR_MODULE_INFERENCEr"   r    extractr!   3ARCHIVED_DATA_FILES_MAX_NUMBER_FOR_MODULE_INFERENCEr   r   r*   )r   r   archived_filesarchive_files_counterr   	extractedr   r   s           rQ   r   r   v  s&    N#  x==!!&)) 
	!Q&!$v'```688@@JJDQQI  yDXXXPfPP   N ! q qN q q qqq 5(44Q77:1=...'448Orf   c                   .    e Zd ZU eed<   eed<   eed<   dS )DatasetModuler^   r   builder_kwargsN)__name__
__module____qualname__r   __annotations__dictr   rf   rQ   r   r     s3         
IIIrf   r   c                   $    e Zd ZU eed<   eed<   dS )MetricModuler^   r   N)r   r   r   r   r   r   rf   rQ   r  r    s%         
IIIIIrf   r  c                       e Zd ZdefdZdS )_DatasetModuleFactoryrS   c                     t           NNotImplementedErrorselfs    rQ   
get_modulez _DatasetModuleFactory.get_module      !!rf   N)r   r   r   r   r
  r   rf   rQ   r  r    s/        "M " " " " " "rf   r  c                       e Zd ZdefdZdS )_MetricModuleFactoryrS   c                     t           r  r  r  s    rQ   r
  z_MetricModuleFactory.get_module  r  rf   N)r   r   r   r  r
  r   rf   rQ   r  r    s/        "L " " " " " "rf   r  c                       e Zd ZdZ ed          	 	 	 	 ddedeeeef                  dee	         deee
ef                  dee         f
d	            Zdee         d
efdZd
efdZdS )GithubMetricModuleFactoryu   Get the module of a metric. The metric script is downloaded from GitHub.

    <Deprecated version="2.5.0">

    Use the new library 🤗 Evaluate instead: https://huggingface.co/docs/evaluate

    </Deprecated>
    O   Use the new library 🤗 Evaluate instead: https://huggingface.co/docs/evaluateNrC   revisionr   r   rP   c                 *   || _         || _        |r|                                nt                      | _        | j        j        dk     rd| j        _        || _        || _        | j                             d          dk    sJ t          |d           d S )N   r   r   metricrw   )
rC   r  r   r   r   max_retriesr   rP   countr~   r	  rC   r  r   r   rP   s         rQ   __init__z"GithubMetricModuleFactory.__init__  s     	 9H^33555nN^N^+a///0D ,*$8!ys##q((((D999999rf   rS   c                     t          | j        | j        dz   |d          }| j                                        }|j        d|_        t          ||          S )Nry   FrK   rC   r  r_   Downloading builder scriptr   )r5   rC   r   r   r   r3   )r	  r  rs   r   s       rQ   download_loading_scriptz1GithubMetricModuleFactory.download_loading_script  s[    !tyty57HS[ejkkk	.3355(0,HO)9oFFFFrf   c           	      .   | j         }	 |                     |          }| j         }nM# t          $ r@ | d}|                     |          }t                              d| j         d           Y nw xY wt          |          }t          | j        t          | j        d|d          || j	                  }| j
        r| j
        nt                      }t          ||g |d| j        | j        	          \  }}t          j                     t!          ||          S )
Nmainz-Couldn't find a directory or a metric named 'zH' in this version. It was picked from the main branch on github instead. Fr  rC   r   r   r   metricsr   r   r   rP   r   rC   r   )r  r  FileNotFoundErrorr   warningrC   r@   r   r5   r   rP   rR   r   r   rU   invalidate_cachesr  )r	  r  r   r   r   rP   r^   r   s           rQ   r
  z$GithubMetricModuleFactory.get_module  s^   =	55h??J}HH  		 		 		#!!99(CC
MDI M M M    		 j))4#hX]^^^ 0	
 
 
 =A<Uqt88[o[q[q3!'!5&,
 
 
T 	#%%%K...s   & AA0/A0NNNN)r   r   r   __doc__r0   r   r	   r   rA   r   r   r  r  r  r
  r   rf   rQ   r  r    s          Zabb 3748<@.2: :: 5g./: ".1	:
  lC&7 89: 'sm: : : cb:$G G# G G G G$/L $/ $/ $/ $/ $/ $/rf   r  c                       e Zd ZdZ ed          	 	 	 ddedee         deee	ef                  dee         fd            Z
d	efd
ZdS )LocalMetricModuleFactoryu   Get the module of a local metric. The metric script is loaded from a local script.

    <Deprecated version="2.5.0">

    Use the new library 🤗 Evaluate instead: https://huggingface.co/docs/evaluate

    </Deprecated>
    r  NrK   r   r   rP   c                     || _         t          |          j        | _        |pt	                      | _        || _        || _        d S r  rK   r   stemrC   r   r   r   rP   r	  rK   r   r   rP   s        rQ   r  z!LocalMetricModuleFactory.__init__  sC     	JJO	.B.2B2B*$8!!!rf   rS   c           	      z   t          | j                  }t          | j        t	          t          | j                  j                  || j                  }| j        r| j        nt                      }t          | j        |g |d| j        | j                  \  }}t          j                     t          ||          S )Nr"  r#  r$  )r@   rK   r   rC   r   r   r   r   rP   rR   r   r   rU   r'  r  )r	  r   r   rP   r^   r   s         rQ   r
  z#LocalMetricModuleFactory.get_module  s    di((4$ty//011 0	
 
 
 =A<Uqt88[o[q[q3y'!5&,
 
 
T 	#%%%K...rf   NNN)r   r   r   r)  r0   r   r	   r   r   r   r  r  r
  r   rf   rQ   r+  r+    s          Zabb 59<@.29 99 ".19  lC&7 89	9
 'sm9 9 9 cb9/L / / / / / /rf   r+  c            
       r    e Zd ZdZ	 	 	 d
dedee         deeeef                  dee         fdZ	de
fd	ZdS )#LocalDatasetModuleFactoryWithScriptzTGet the module of a local dataset. The dataset script is loaded from a local script.NrK   r   r   rP   c                     || _         t          |          j        | _        |pt	                      | _        || _        || _        d S r  r-  r/  s        rQ   r  z,LocalDatasetModuleFactoryWithScript.__init__1  sC     	JJO	.B.2B2B*$8!!!rf   rS   c           	      0   t          | j                  j        t          j        z  }t          | j                  j        dz  }t          | j                  }t          | j        t          t          | j                  j                  || j	                  }g }|
                                r.|                    t          j        t          |          f           |
                                r|                    d|f           | j        r| j        nt                      }t          | j        |||d| j        | j                  \  }}t!          j                     |t          t          | j                  j                  d}	t%          |||	          S )N	README.mdr"  datasetsr$  )r   r   )r   rK   r   r   DATASETDICT_INFOS_FILENAMEr@   r   rC   r   r   is_filerp   rP   rR   r   r   rU   r'  r   )
r	  dataset_infos_pathdataset_readme_pathr   r   r   rP   r^   r   r   s
             rQ   r
  z.LocalDatasetModuleFactoryWithScript.get_module>  s~   !$)__3f6WW"49oo4{Bdi((4$ty//011 0	
 
 
 %%'' 	b##V%FL^H_H_$`aaa&&(( 	H##[2E$FGGG<@<Uqt88[o[q[q3y'-!5',
 
 
T 	#%%%"&Sdi9O5P5PQQ[$???rf   r1  )r   r   r   r)  r   r	   r   r   r   r  r   r
  r   rf   rQ   r3  r3  .  s        ^^
 59<@.29 99 ".19  lC&7 89	9
 'sm9 9 9 9@M @ @ @ @ @ @rf   r3  c                       e Zd ZdZ	 	 	 d
dedee         deeeeef                  deee	ef                  fdZ
defd	ZdS )&LocalDatasetModuleFactoryWithoutScriptzGet the module of a dataset loaded from the user's data files. The dataset builder module to use is inferred
    from the data files extensions.NrK   data_dir
data_filesr   c                     |r1t           j                            |          rt          d|           || _        t	          |          j        | _        || _        || _        || _	        d S )Nz;`data_dir` must be relative to a dataset directory's root: )
rJ   rK   isabsr   r   r.  rC   r?  r>  r   )r	  rK   r>  r?  r   s        rQ   r  z/LocalDatasetModuleFactoryWithoutScript.__init__c  sm      	ch// 	ca[_aabbb	JJO	$ *rf   rS   c                    | j         r*t          j                            | j        | j                   n| j        }| j        t          | j                  nt          |          }t          j        ||t                    }d |
                                D             }t          t          |                                                    \  t          fd|                                D                       rt          d|           st!          d| j                   | j        t"          v rz|t$          k    ro	 t'          |          }n# t           $ r d }Y nw xY w|Jt)          j        ||          }|D ]1}t)          ||         |z   ||         j        |j        z             ||<   2t,                   \  }}	|	|t          j                            | j                            d                    | j        dt          j                            t          j                            | j        t4          j                            rt9          t          j                            | j        t4          j                  d	
          5 }
t;          j        |
          }d d d            n# 1 swxY w Y   |rbt          t          |                    d<   t?          j         t          t          |                                                              d<   t          j                            t          j                            | j        d                    rtC          j"        tG          | j                  dz            }tI          |%                    d          tL                    r=|d         r5|d         d         }t?          j'        |          d<   d|v r|d         d<   n^tI          |%                    d          tP                    r6|d         r.|d         }t?          j'        |          d<   d|v r|d         d<   tS          ||	          S )N)r   allowed_extensionsc                 4    i | ]\  }}|t          |          S r   )r   )r   r   r   s      rQ   
<dictcomp>zELocalDatasetModuleFactoryWithoutScript.get_module.<locals>.<dictcomp>}  s4     
 
 
DZE?E.??
 
 
rf   c              3   (   K   | ]}f|k    V  d S r  r   r   split_moduler   module_names     rQ   r   zDLocalDatasetModuleFactoryWithoutScript.get_module.<locals>.<genexpr>  -      hh^,<hhhhhhrf   =Couldn't infer the same data file format for all splits. Got 5No (supported) data files or dataset script found in )r   r   )r   r?  config_namer   ri   rj   rM  infor6  dataset_infor   )*r>  rJ   rK   rL   r?  r   r   r   from_local_or_remoteALL_ALLOWED_EXTENSIONSrX   nextiterr   anyr   r%  r+   r   r   r   origin_metadatar,   r   rstripr   r   r8  rO   r   loadr&   	from_dictr?   from_readmer   
isinstancegetrn   _from_yaml_dictr   r   )r	  r   patternsr?  split_modulesmetadata_patternsmetadata_fileskeyr^   r   ru   dataset_infosdataset_metadatadataset_info_dictr   rI  s                 @@rQ   r
  z1LocalDatasetModuleFactoryWithoutScript.get_modules  s   >BmZBGLLDM:::QUQZ	26/2Mdo...SlmvSwSw 	 #75
 
 



 
^h^n^n^p^p
 
 
 '+40D0D0F0F+G+G&H&H#^hhhhhQ^QeQeQgQghhhhh 	nl]jllmmm 	i#$g\`\e$g$ghhh?"{6O'O'OT\`tTtTt)$A)$L$L!!$ ) ) )$(!!!) ,!.!CDUaj!k!k!k%  C&3"3.8"37.:XX' 'JsOO 7{CT$7++DI,<,<S,A,ABB	
 

 
 7>>"',,ty&2STTUU 	cbgll49f.OPP[bccc ?gh26)A,,? ? ? ? ? ? ? ? ? ? ? ? ? ? ? c04T-5H5H0I0I}-)4)>tDI]I]I_I_D`D`?a?a)b)bv&7>>"',,ty+>>?? 	U.:4	??[;XYY*..~>>EE 	UJZ[iJj 	U$4^$DQ$G!)4)DEV)W)Wv& $5554Em4TN=1,00@@$GG UL\]kLl U$4^$D!)4)DEV)W)Wv& $5554Em4TN=1[$???s$   E E$#E$J11J58J5r1  )r   r   r   r)  r   r	   r   r   r   r   r  r   r
  r   rf   rQ   r=  r=  _  s        ' ' #'7;<@+ ++ 3-+ U3d?34	+
  lC&7 89+ + + + 9@M 9@ 9@ 9@ 9@ 9@ 9@rf   r=  c                       e Zd ZdZ	 	 	 	 ddedee         deeeeef                  dee	         deee
ef                  f
dZd	efd
ZdS )PackagedDatasetModuleFactoryz`Get the dataset builder module from the ones that are packaged with the library: csv, json, etc.NrC   r>  r?  r   r   c                 n    || _         || _        || _        || _        || _        t          |d           d S Nr_   r  )rC   r?  r>  r   r   r~   )r	  rC   r>  r?  r   r   s         rQ   r  z%PackagedDatasetModuleFactory.__init__  sB     	$ .*D	::::::rf   rS   c                    | j         Et          t          | j                                                                                             n,t          t                                                                }| j        t          | j                  nt          |          }t          j	        || j
        j        |          }| j        | j        t          v r|t          k    rz	 t          |          }n# t           $ r d }Y nw xY w|Ut#          j	        || j
        j        |          }|D ]1}t#          ||         |z   ||         j        |j        z             ||<   2t&          | j                 \  }}||d}	t)          |||	          S )N)r   r   )r   r?  )r>  r   r   
expanduserresolver?  r   r   r   rP  r   r   rC   r+   r   r   r%  r   rU  r,   r   )
r	  r   r]  r?  r_  r`  ra  r^   r   r   s
             rQ   r
  z'PackagedDatasetModuleFactory.get_module  s   ?C}?XCT]##..0088::;;;^abfbhbhbpbpbrbr^s^s 	 37/2Mdo...SlmvSwSw 	 #7/>
 
 


 ?"ty4M'M'MRZ^rRrRr)$A)$L$L!!$ ) ) )$(!!!) ,!.!C%d6J6Yen" " " &  C&3"3.8"37.:XX' 'JsOO 7tyAT"&jAA[$???s   &C6 6DDr(  )r   r   r   r)  r   r	   r   r   r   r   r   r  r   r
  r   rf   rQ   rf  rf    s        jj
 #'7;48<@; ;; 3-; U3d?34	;
 ".1;  lC&7 89; ; ; ;@M @ @ @ @ @ @rf   rf  c                       e Zd ZdZ	 	 	 	 	 ddedeeeef                  dee         deeeee	f                  dee
         deeeef                  fd	Zd
efdZdS )$HubDatasetModuleFactoryWithoutScriptz
    Get the module of a dataset loaded from data files of a dataset repository.
    The dataset builder module to use is inferred from the data files extensions.
    NrC   r  r>  r?  r   r   c                     || _         || _        || _        || _        |pt	                      | _        || _        t          |d           d S rh  )rC   r  r?  r>  r   r   r   r~   )r	  rC   r  r>  r?  r   r   s          rQ   r  z-HubDatasetModuleFactoryWithoutScript.__init__  sT     	 $ .B.2B2B*D	::::::rf   rS   c                 (	    t          t          j                                       j         j         j        j        d          } j        t           j                  nt          | j                  }t          j        || j        t                    } fd|                                D             }t!          t#          |                                                    \  t'          fd|                                D                       rt)          d|           st+          d j                    j        t,          v r|t.          k    r{	 t1          | j                  }n# t*          $ r d }Y nw xY w|Pt3          j        || j                  }|D ]1}t3          ||         |z   ||         j        |j        z             ||<   2t6                   \  }}	|	| j                            d	d
          t;           j        d j                   j        d j                                        }
|
j        d|
_        	 tA          t;           j        t          j!         j                  |
          }tE          |d          5 }tG          j$        |          }d d d            n# 1 swxY w Y   |rbt!          t#          |                    d<   tK          j&        t!          t#          |                                                              d<   n# t*          $ r Y nw xY w j                                        }
|
j        d|
_        	 tA          t;           j        d j                  |
          }tO          j(        tS          |                    }tU          |+                    d          tX                    r=|d         r5|d         d         }tK          j-        |          d<   d|v r|d         d<   n^tU          |+                    d          t\                    r6|d         r.|d         }tK          j-        |          d<   d|v r|d         d<   n# t*          $ r Y nw xY wt_          ||	          S )N      Y@)r  tokentimeout)rO  r   rC  c                 N    i | ]!\  }}|t          |j        j                   "S )r   )r   r   r   )r   r   r   r	  s      rQ   rE  zCHubDatasetModuleFactoryWithoutScript.get_module.<locals>.<dictcomp>	  sC     
 
 
& .tOcOrsss
 
 
rf   c              3   (   K   | ]}f|k    V  d S r  r   rG  s     rQ   r   zBHubDatasetModuleFactoryWithoutScript.get_module.<locals>.<genexpr>  rJ  rf   rK  rL  )rO  r   r   r   r!  r  )r   r?  rM  r   repo_idDownloading metadatar   ri   rj   rM  rN  Downloading readmer6  rO  r   )0r   r   HF_ENDPOINTrO  rC   r  r   r   r?  r   r   r>  r   from_hf_reporQ  rX   rR  rS  r   rT  r   r%  r+   r   r   r   rU  r,   r   r;   r   r   r3   r8  rO   r   rW  r&   rX  r?   rY  r   rZ  r[  rn   r\  r   r   )r	  hfh_dataset_infor]  r?  r^  r_  r`  ra  r^   r   r   r:  ru   rb  r;  rc  rd  r   rI  s   `                @@rQ   r
  z/HubDatasetModuleFactoryWithoutScript.get_module  se    !344AAI]&5	 B 
 
 * do...89I4=YY 	
 #/)m5	
 
 


 
 
 
*4*:*:*<*<
 
 
 '+40D0D0F0F+G+G&H&H#^hhhhhQ^QeQeQgQghhhhh 	nl]jllmmm 	i#$g\`\e$g$ghhh?"{6O'O'OT\`tTtTt)$OP`bfbo$p$p!!$ ) ) )$(!!!) ,!.!;%4DPTP]" " " &  C&3"3.8"37.:XX' 'JsOO 7{CT$9,,S$77#DIrDMJJJy
 
 
 .3355(0,BO)	!,49f&GRVR_``` /" " " (7;;; ?q26)A,,? ? ? ? ? ? ? ? ? ? ? ? ? ? ? c04T-5H5H0I0I}-)4)>tDI]I]I_I_D`D`?a?a)b)bv&  	 	 	D	.3355(0,@O)	"-49kDMJJJ /# # #  /:4@S;T;TUU*..~>>EE 	UJZ[iJj 	U$4^$DQ$G!)4)DEV)W)Wv& $5554Em4TN=1,00@@$GG UL\]kLl U$4^$D!)4)DEV)W)Wv& $5554Em4TN=1  	 	 	D	[$???s]    E6 6FFAL+ J;/L+ ;J??L+ J?A'L+ +
L87L8#DQ3 3
R ?R )NNNNN)r   r   r   r)  r   r	   r   rA   r   r   r   r   r  r   r
  r   rf   rQ   rm  rm    s          37"&7;48<@; ;; 5g./; 3-	;
 U3d?34; ".1;  lC&7 89; ; ; ;"X@M X@ X@ X@ X@ X@ X@rf   rm  c                       e Zd ZdZ	 	 	 	 ddedeeeef                  dee         deee	ef                  dee         f
dZ
d	efd
Zd	efdZd	efdZd	efdZdS )!HubDatasetModuleFactoryWithScriptz~Get the module of a dataset from a dataset repository. The dataset script comes from the script inside the dataset repository.NrC   r  r   r   rP   c                     || _         || _        |pt                      | _        || _        || _        t          |d           d S rh  )rC   r  r   r   r   rP   r~   r  s         rQ   r  z*HubDatasetModuleFactoryWithScript.__init__U  sN     	 .B.2B2B*$8!D	::::::rf   rS   c                     t          | j        | j                            d          d         dz   | j                  }| j                                        }|j        d|_        t          ||          S )Nr   r   ry   rv  rK   r  r  r   )r;   rC   r   r  r   r   r   r3   )r	  rs   r   s      rQ   r  z9HubDatasetModuleFactoryWithScript.download_loading_scriptd  sm    tytys7K7KB7ORW7Wbfboppp	.3355(0,HO)9oFFFFrf   c                     t          | j        t          j        | j                  }| j                                        }|j        d|_        	 t          ||          S # t          t          f$ r Y d S w xY w)Nr  rw  r   )r;   rC   r   r8  r  r   r   r   r3   r%  ConnectionError)r	  rb  r   s      rQ   download_dataset_infos_filez=HubDatasetModuleFactoryWithScript.download_dataset_infos_filek  s    "496;\gkgtuuu.3355(0,BO)	 /    "?3 	 	 	44	s   A   A54A5c                     t          | j        d| j                  }| j                                        }|j        d|_        	 t          ||          S # t          t          f$ r Y d S w xY w)Nr6  r  rx  r   )	r;   rC   r  r   r   r   r3   r%  r  )r	  
readme_urlr   s      rQ   download_dataset_readme_filez>HubDatasetModuleFactoryWithScript.download_dataset_readme_filey  s    	dm\\\
.3355(0,@O)	 /    "?3 	 	 	44	s   A A+*A+c           	         |                                  }|                                 }|                                 }t          |          }t	          | j        t          | j        d| j                  || j                  }g }|r!|	                    t          j        |f           |r|	                    d|f           | j        r| j        nt                      }t          ||||d| j        | j                  \  }}	t!          j                     |	t          | j        d| j                  | j        d}
t%          ||	|
          S )	Nr!  r  r"  r6  r7  r$  ru  )r   r   rv  )r  r  r  r@   r   rC   r;   r  r   rp   r   r8  rP   rR   r   r   rU   r'  r   )r	  r   r:  r;  r   r   r   rP   r^   r   r   s              rQ   r
  z,HubDatasetModuleFactoryWithScript.get_module  sl   1133
!==??"??AAj))4 dmTTT 0	
 
 
  	]##V%FHZ$[\\\ 	H##[2E$FGGG<@<Uqt88[o[q[q3!'-!5',
 
 
T 	#%%%#DIrDMJJJy
 

 [$???rf   r(  )r   r   r   r)  r   r	   r   rA   r   r   r  r  r  r  r   r
  r   rf   rQ   r}  r}  R  s        I  I
 3748<@.2; ;; 5g./; ".1	;
  lC&7 89; 'sm; ; ; ;G G G G GS    c    #@M #@ #@ #@ #@ #@ #@rf   r}  c                   >    e Zd ZdZ	 ddedee         fdZdefdZdS )	CachedDatasetModuleFactoryz
    Get the module of a dataset that has been loaded once already and cached.
    The script that is loaded from the cache is the most recent one with a matching name.
    NrC   rP   c                 b    || _         || _        | j                             d          dk    sJ d S )Nr   r   rC   rP   r  r	  rC   rP   s      rQ   r  z#CachedDatasetModuleFactory.__init__  s8    
 	$8!ys##q((((((rf   rS   c           	           j         r j         nt                      }t          j                            |d j                            dd                    t          j                                      rd t          j                  D             nd }|st          d j         d|            fd}t          ||          d	         }d
t          j                            |           dt          j         ||                     d j         d}t          j        s|dz  }t                              |           d                    t          j                            |          d j                            dd          | j                            d          d	         g          }t'          j                     | j        d}t+          |||          S )Nr7  r   r   c                 8    g | ]}t          |          d k    |S @   r   r   hs     rQ   r   z9CachedDatasetModuleFactory.get_module.<locals>.<listcomp>  #    NNN1Q2Qrf   zDataset  is not cached in c                     t                    | z  j                            d          d         dz   z                                  j        S )Nr   r   ry   )r   rC   r   statst_mtimemodule_hashr   r	  s    rQ   _get_modification_timezECachedDatasetModuleFactory.get_module.<locals>._get_modification_time  sF    233kATY__UXEYEYZ\E]`eEefllnnwwrf   ra  r   3Using the latest cached version of the module from  (last modified on () since it couldn't be found locally at r   &, or remotely on the Hugging Face Hub.)r   rv  )rP   rR   rJ   rK   rL   rC   r   rl   listdirr%  sortedtimectimer   r{   r   r&  r   r   rU   r'  r   )	r	  rP   hashesr  r   warning_msgr^   r   r   s	   `       @rQ   r
  z%CachedDatasetModuleFactory.get_module  s   <@<Uqt88[o[q[q$&GLL1EzSWS\SdSdehjnSoSo$p$p! w}}677NN
#<==NNNN 	
  	d#$bty$b$bL`$b$bccc	x 	x 	x 	x 	x 	x f"8999"=9"',,OhjnBoBo 9 9!%,B,B4,H,H!I!I9 9,0I9 9 9 	
 ) 	DCCK{###hh  !566	!!#t,,	$$R(
 
 	#%%%y
 
 [$???rf   r  )	r   r   r   r)  r   r	   r  r   r
  r   rf   rQ   r  r    st          /3) )) 'sm) ) ) )'@M '@ '@ '@ '@ '@ '@rf   r  c                   `    e Zd ZdZ ed          	 d	dedee         fd            ZdefdZ	dS )
CachedMetricModuleFactoryu6  
    Get the module of a metric that has been loaded once already and cached.
    The script that is loaded from the cache is the most recent one with a matching name.

    <Deprecated version="2.5.0">

    Use the new library 🤗 Evaluate instead: https://huggingface.co/docs/evaluate

    </Deprecated>
    r  NrC   rP   c                 b    || _         || _        | j                             d          dk    sJ d S )Nr   r   r  r  s      rQ   r  z"CachedMetricModuleFactory.__init__  s8     	$8!ys##q((((((rf   rS   c           
           j         r j         nt                      }t          j                            |d j                  t          j                                      rd t          j                  D             nd }|st          d j         d|            fd}t          ||          d         }t                              dt          j                            |           d	t          j         ||                     d
 j         d           d                    t          j                            |          d j        | j        g          }t          j                     t#          ||          S )Nr#  c                 8    g | ]}t          |          d k    |S r  r  r  s     rQ   r   z8CachedMetricModuleFactory.get_module.<locals>.<listcomp>   r  rf   zMetric r  c                 l    t                    | z  j        dz   z                                  j        S )Nry   )r   rC   r  r  r  s    rQ   r  zDCachedMetricModuleFactory.get_module.<locals>._get_modification_time  s3    233kATYQVEVW]]__hhrf   r  r   r  r  r  r  r   )rP   rR   rJ   rK   rL   rC   rl   r  r%  r  r   r&  r  r  r   rU   r'  r  )r	  rP   r  r  r   r^   r   s   `     @rQ   r
  z$CachedMetricModuleFactory.get_module  s   <@<Uqt88[o[q[q$&GLL1EyRVR[$\$\! w}}677NN
#<==NNNN 	
  	c#$adi$a$aK_$a$abbb	i 	i 	i 	i 	i 	i f"8999"=^"',,OhjnBoBo ^ ^!%,B,B4,H,H!I!I^ ^,0I^ ^ ^	
 	
 	
 hh 0 01E F F	SWS\^bdhdmnoo#%%%K...rf   r  )
r   r   r   r)  r0   r   r	   r  r  r
  r   rf   rQ   r  r    s        	 	 Zabb /3) )) 'sm) ) ) cb)/L / / / / / /rf   r  rK   r  r>  r?  c                 n	   |t          d"i |}t          |pt          j                  }d|_        d|_        |t          j        k    |_        t          t          d | 	                    t          j        d                              d                              d         }|                    d          s|dz   }t          j                            | |          }	| t           v r&t#          | ||||                                          S |                     |          rbt          j                            |           r$t)          | ||                                          S t+          d	t-          |                      t          j                            |	          r$t)          |	||                                          S t          j                            |           r%t1          | |||
                                          S t3          |           r~|                     d          dk    rd	 t7                       t9          t:          j                  }
	 |
                    | ||j         d          }n# tB          $ r}tE          |tF          tH          j%        j&        tH          j%        j'        f          r(tO          d|  dtQ          |          j)         d          dtU          |          v r d|  d}t+          |r	|d| dz   n|          dtU          |          v r%d|  d}|r	|d| dz   n|}t+          |dz             |d}~ww xY w|d |j+        D             v r&tY          | ||||                                          S t[          | |||||                                          S # tB          $ r}	 t]          | |                                          cY d}~S # tB          $ r}tE          |tF                    rtO          d|  d|           dtE          |t^                    r|dtE          |t*                    r;t+          d	t-          |	           d|  dtQ          |          j)         d |           d|dd}~ww xY wd}~ww xY wt+          d	t-          |	           d!          )#a#  
    Download/extract/cache a dataset module.

    Dataset codes are cached inside the dynamic modules cache to allow easy import (avoid ugly sys.path tweaks).

    Args:

        path (str): Path or name of the dataset.
            Depending on ``path``, the dataset builder that is used comes from a generic dataset script (JSON, CSV, Parquet, text etc.) or from the dataset script (a python file) inside the dataset directory.

            For local datasets:

            - if ``path`` is a local directory (containing data files only)
              -> load a generic dataset builder (csv, json, text etc.) based on the content of the directory
              e.g. ``'./path/to/directory/with/my/csv/data'``.
            - if ``path`` is a local dataset script or a directory containing a local dataset script (if the script has the same name as the directory):
              -> load the dataset builder from the dataset script
              e.g. ``'./dataset/squad'`` or ``'./dataset/squad/squad.py'``.

            For datasets on the Hugging Face Hub (list all available datasets and ids with ``datasets.list_datasets()``)

            - if ``path`` is a dataset repository on the HF hub (containing data files only)
              -> load a generic dataset builder (csv, text etc.) based on the content of the repository
              e.g. ``'username/dataset_name'``, a dataset repository on the HF hub containing your data files.
            - if ``path`` is a dataset repository on the HF hub with a dataset script (if the script has the same name as the directory)
              -> load the dataset builder from the dataset script in the dataset repository
              e.g. ``glue``, ``squad``, ``'username/dataset_name'``, a dataset repository on the HF hub containing a dataset script `'dataset_name.py'`.

        revision (:class:`~utils.Version` or :obj:`str`, optional): Version of the dataset script to load.
            As datasets have their own git repository on the Datasets Hub, the default version "main" corresponds to their "main" branch.
            You can specify a different version than the default "main" by using a commit SHA or a git tag of the dataset repository.
        download_config (:class:`DownloadConfig`, optional): Specific download configuration parameters.
        download_mode (:class:`DownloadMode` or :obj:`str`, default ``REUSE_DATASET_IF_EXISTS``): Download/generate mode.
        dynamic_modules_path (Optional str, defaults to HF_MODULES_CACHE / "datasets_modules", i.e. ~/.cache/huggingface/modules/datasets_modules):
            Optional path to the directory in which the dynamic modules are saved. It must have been initialized with :obj:`init_dynamic_modules`.
            By default, the datasets and metrics are stored inside the `datasets_modules` module.
        data_dir (:obj:`str`, optional): Directory with the data files. Used only if `data_files` is not specified,
            in which case it's equal to pass `os.path.join(data_dir, "**")` as `data_files`.
        data_files (:obj:`Union[Dict, List, str]`, optional): Defining the data_files of the dataset configuration.
        **download_kwargs (additional keyword arguments): optional attributes for DownloadConfig() which will override
            the attributes in download_config if supplied.

    Returns:
        DatasetModule
    NTc                     | S r  r   xs    rQ   <lambda>z(dataset_module_factory.<locals>.<lambda>U  s    Q rf   r   r   ry   )r>  r?  r   r   r   rP   z"Couldn't find a dataset script at )r>  r?  r   r   rp  )rv  r  r   rr  zCouldn't reach 'z' on the Hub ()404z	Dataset 'z' doesn't exist on the Hubz at revision ''401zT. If the repo is private or gated, make sure to log in with `huggingface-cli login`.c                     g | ]	}|j         
S r   )	rfilename)r   siblings     rQ   r   z*dataset_module_factory.<locals>.<listcomp>  s    SSS'G-SSSrf   r  r   r   rP   )r  r>  r?  r   r   rP   z1Couldn't reach the Hugging Face Hub for dataset 'z': z8 or any data file in the same directory. Couldn't find 'z"' on the Hugging Face Hub either: r   z( or any data file in the same directory.r   )0r   r   REUSE_DATASET_IF_EXISTSextract_compressed_fileforce_extractr   force_downloadrn   filterr   rJ   sepr   r   rK   rL   r,   rf  r
  r   r3  r%  r8   rl   r=  r7   r  r2   r   r   ry  rO  r   r}   rZ  r1   requests
exceptionsConnectTimeoutr  typer   r   siblingsr}  rm  r  r   )rK   r  r   r   rP   r>  r?  download_kwargsrz   combined_pathhf_apirO  emsge1e2s                   rQ   dataset_module_factoryr    s   n (;;?;; !V,2VWWM.2O+$(O!%2l6S%SO"F;;RVS(A(A(G(G(L(LMMNNrRHU## $e#GLLx00M$ )))+!+'
 
 
 *,,	 
x	 	  T
7>>$ 	l6MH\  jll $$jIbcgIhIh$j$jkkk		&	& M
2Ma
 
 

*,,	 
t		 I
58
-
 
 

*,,	 
$		 D
DJJsOOq$8$8?	#-///6-..F%22 %#2#A!	  3        , +: +;   **dT*d*dQUVWQXQXQa*d*d*deeec!ff__FdFFFC+RZ,cC2N82N2N2N,N,N`cdddc!ff__FdFFFC@HQ# < < < <<<cC+tt   G)* SS\=RSSSSS8%$3"/)=   *,, <%%)$3"/   *,, 	# 	# 	##1$Mabbbmmoooooooo 
# 
# 
#b"677 w)*k^b*k*kgi*k*kllrvvb"344 '$&b"344  +l=VWd=e=e l l*.l lRVWYRZRZRcl lgil l     d"
#	#"   D1J=1Y1Y  D  D  D
 
 	
sb   4'N4 I; :N4 ;
MCMM;N4 &N4 4
R?"O'!R'
R1BRRRRr  c           
         t          j                    5  t          j        ddt                     |t	          di |}t          |pt
          j                  }d|_        d|_        t          t          d |                     t          j        d                              d                              d         }|                    d	          s|d	z   }t          j                            | |          }|                     |          rnt          j                            |           r0t'          | ||
                                          cddd           S t+          dt-          |                      t          j                            |          r0t'          |||
                                          cddd           S t/          |           r|                     d          dk    r	 t3          | ||||                                          	 cddd           S # t4          $ r}	 t7          | |                                          cY d}~cddd           S # t4          $ rA}	t9          |t*                    s|dt+          dt-          |           d|  d          dd}	~	ww xY wd}~ww xY wt+          dt-          |           d          # 1 swxY w Y   dS )u  
    Download/extract/cache a metric module.

    <Deprecated version="2.5.0">

    Use the new library 🤗 Evaluate instead: https://huggingface.co/docs/evaluate

    </Deprecated>

    Metrics codes are cached inside the dynamic modules cache to allow easy import (avoid ugly sys.path tweaks).

    Args:

        path (str): Path or name of the metric script.

            - if ``path`` is a local metric script or a directory containing a local metric script (if the script has the same name as the directory):
              -> load the module from the metric script
              e.g. ``'./metrics/accuracy'`` or ``'./metrics/accuracy/accuracy.py'``.
            - if ``path`` is a metric on the Hugging Face Hub (ex: `glue`, `squad`)
              -> load the module from the metric script in the GitHub repository at huggingface/datasets
              e.g. ``'accuracy'`` or ``'rouge'``.

        revision (Optional ``Union[str, datasets.Version]``):
            If specified, the module will be loaded from the datasets repository at this version.
            By default:
            - it is set to the local version of the lib.
            - it will also try to load it from the main branch if it's not available at the local version of the lib.
            Specifying a version that is different from your local version of the lib might cause compatibility issues.
        download_config (:class:`DownloadConfig`, optional): Specific download configuration parameters.
        download_mode (:class:`DownloadMode` or :obj:`str`, default ``REUSE_DATASET_IF_EXISTS``): Download/generate mode.
        dynamic_modules_path (Optional str, defaults to HF_MODULES_CACHE / "datasets_modules", i.e. ~/.cache/huggingface/modules/datasets_modules):
            Optional path to the directory in which the dynamic modules are saved. It must have been initialized with :obj:`init_dynamic_modules`.
            By default, the datasets and metrics are stored inside the `datasets_modules` module.
        **download_kwargs (additional keyword arguments): optional attributes for DownloadConfig() which will override
            the attributes in download_config if supplied.

    Returns:
        MetricModule
    ignore'.*https://huggingface.co/docs/evaluate$messagecategoryNTc                     | S r  r   r  s    rQ   r  z'metric_module_factory.<locals>.<lambda>  s     rf   r   r   ry   r  z!Couldn't find a metric script at r   r  r  z
. Metric 'z/' doesn't exist on the Hugging Face Hub either.r   r   )warningscatch_warningsfilterwarningsFutureWarningr   r   r  r  r  rn   r  r   rJ   r  r   r   rK   rL   r   r+  r
  r%  r8   r7   r  r  r}   r  rZ  )
rK   r  r   r   rP   r  rz   r  r  r  s
             rQ   metric_module_factoryr    s   ` 
	 	"	" 0u 0u2[fstttt",????O$]%Zl6Z[[26/(,%{{DLL,E,E,K,KC,P,PQQRRSUV  '' 	(%'HT844=="" !	uw~~d## o/L`  *,,'0u 0u 0u 0u 0u 0u 0u 0u* ((mLefjLkLk(m(mnnnW^^M** 	u+]Qe  jll30u 0u 0u 0u 0u 0u 0u 0u4 d## 	u

31(<(< 0%$3"/)=   *,,90u 0u 0u 0u 0u 0u 0u 0uH       4TPdeeepprrrrrrrO0u 0u 0u 0u 0u 0u 0u 0uP !      %b*;<< + d*+Y<UVc<d<d Y Y#'Y Y Y       $$sHaboHpHp$s$s$sttta0u 0u 0u 0u 0u 0u 0u 0u 0u 0usa   D:KA!K	(K2%H%%
J70"I$J7K$
J/.<J**J//J22J77#KK!Kue   Use 'evaluate.load' instead, from the new library 🤗 Evaluate: https://huggingface.co/docs/evaluateFrM  
process_idnum_process	cache_direxperiment_idkeep_in_memoryc
                 j   t          j                    5  t          j        ddt                     t	          |pt          j                  }t          | |	||          j        }t          |d          } |d
||||||d|
}|	                    |           |cd	d	d	           S # 1 swxY w Y   d	S )u(	  Load a `datasets.Metric`.

    <Deprecated version="2.5.0">

    Use `evaluate.load` instead, from the new library 🤗 Evaluate: https://huggingface.co/docs/evaluate

    </Deprecated>

    Args:

        path (``str``):
            path to the metric processing script with the metric builder. Can be either:
                - a local path to processing script or the directory containing the script (if the script has the same name as the directory),
                    e.g. ``'./metrics/rouge'`` or ``'./metrics/rogue/rouge.py'``
                - a metric identifier on the HuggingFace datasets repo (list all available metrics with ``datasets.list_metrics()``)
                    e.g. ``'rouge'`` or ``'bleu'``
        config_name (:obj:`str`, optional): selecting a configuration for the metric (e.g. the GLUE metric has a configuration for each subset)
        process_id (:obj:`int`, optional): for distributed evaluation: id of the process
        num_process (:obj:`int`, optional): for distributed evaluation: total number of processes
        cache_dir (Optional str): path to store the temporary predictions and references (default to `~/.cache/huggingface/metrics/`)
        experiment_id (``str``): A specific experiment id. This is used if several distributed evaluations share the same file system.
            This is useful to compute metrics in distributed setups (in particular non-additive metrics like F1).
        keep_in_memory (bool): Whether to store the temporary results in memory (defaults to False)
        download_config (Optional ``datasets.DownloadConfig``: specific download configuration parameters.
        download_mode (:class:`DownloadMode` or :obj:`str`, default ``REUSE_DATASET_IF_EXISTS``): Download/generate mode.
        revision (Optional ``Union[str, datasets.Version]``): if specified, the module will be loaded from the datasets repository
            at this version. By default, it is set to the local version of the lib. Specifying a version that is different from
            your local version of the lib might cause compatibility issues.

    Returns:
        `datasets.Metric`

    Example:

    ```py
    >>> from datasets import load_metric
    >>> accuracy = load_metric('accuracy')
    >>> accuracy.compute(references=[1, 0], predictions=[1, 1])
    {'accuracy': 0.5}
    ```
    r  r  r  )r  r   r   F)r_   )rM  r  r  r  r  r  r   Nr   )
r  r  r  r  r   r  r  r^   re   download_and_prepare)rK   rM  r  r  r  r  r  r   r   r  metric_init_kwargsmetric_module
metric_clsr  s                 rQ   load_metricr  -  s2   n 
	 	"	"  2[fstttt$]%Zl6Z[[-8_Ta
 
 

 	 '}eDDD
 
#!#)'
 
 !
 
 	##O#DDD-                 s   BB((B,/B,featuresstorage_optionsc                 (    t          |pt           j                  }|	+|r|                                nt                      }|	|_        t           |||||          }t          |j                  }|j        }|	                    d|          }|	                    d|          }|	                    d          } t          v r;|9d  d} fdt          D             }|r|d	|d
          dz  }t          |           |d|||||||	|
d||}|S )a  Load a dataset builder from the Hugging Face Hub, or a local dataset. A dataset builder can be used to inspect general information that is required to build a dataset (cache directory, config, dataset info, etc.)
    without downloading the dataset itself.

    You can find the list of datasets on the [Hub](https://huggingface.co/datasets) or with [`datasets.list_datasets`].

    A dataset is a directory that contains:

    - some data files in generic formats (JSON, CSV, Parquet, text, etc.)
    - and optionally a dataset script, if it requires some code to read the data files. This is used to load any kind of formats or structures.

    Note that dataset scripts can also download and read data files from anywhere - in case your data files already exist online.

    Args:

        path (`str`):
            Path or name of the dataset.
            Depending on `path`, the dataset builder that is used comes from a generic dataset script (JSON, CSV, Parquet, text etc.) or from the dataset script (a python file) inside the dataset directory.

            For local datasets:

            - if `path` is a local directory (containing data files only)
              -> load a generic dataset builder (csv, json, text etc.) based on the content of the directory
              e.g. `'./path/to/directory/with/my/csv/data'`.
            - if `path` is a local dataset script or a directory containing a local dataset script (if the script has the same name as the directory)
              -> load the dataset builder from the dataset script
              e.g. `'./dataset/squad'` or `'./dataset/squad/squad.py'`.

            For datasets on the Hugging Face Hub (list all available datasets and ids with [`datasets.list_datasets`])

            - if `path` is a dataset repository on the HF hub (containing data files only)
              -> load a generic dataset builder (csv, text etc.) based on the content of the repository
              e.g. `'username/dataset_name'`, a dataset repository on the HF hub containing your data files.
            - if `path` is a dataset repository on the HF hub with a dataset script (if the script has the same name as the directory)
              -> load the dataset builder from the dataset script in the dataset repository
              e.g. `glue`, `squad`, `'username/dataset_name'`, a dataset repository on the HF hub containing a dataset script `'dataset_name.py'`.

        name (`str`, *optional*):
            Defining the name of the dataset configuration.
        data_dir (`str`, *optional*):
            Defining the `data_dir` of the dataset configuration. If specified for the generic builders (csv, text etc.) or the Hub datasets and `data_files` is `None`,
            the behavior is equal to passing `os.path.join(data_dir, **)` as `data_files` to reference all the files in a directory.
        data_files (`str` or `Sequence` or `Mapping`, *optional*):
            Path(s) to source data file(s).
        cache_dir (`str`, *optional*):
            Directory to read/write data. Defaults to `"~/.cache/huggingface/datasets"`.
        features ([`Features`], *optional*):
            Set the features type to use for this dataset.
        download_config ([`DownloadConfig`], *optional*):
            Specific download configuration parameters.
        download_mode ([`DownloadMode`] or `str`, defaults to `REUSE_DATASET_IF_EXISTS`):
            Download/generate mode.
        revision ([`Version`] or `str`, *optional*):
            Version of the dataset script to load.
            As datasets have their own git repository on the Datasets Hub, the default version "main" corresponds to their "main" branch.
            You can specify a different version than the default "main" by using a commit SHA or a git tag of the dataset repository.
        use_auth_token (`str` or `bool`, *optional*):
            Optional string or boolean to use as Bearer token for remote files on the Datasets Hub.
            If `True`, or not specified, will get token from `"~/.huggingface"`.
        storage_options (`dict`, *optional*, defaults to `None`):
            **Experimental**. Key/value pairs to be passed on to the dataset file-system backend, if any.

            <Added version="2.11.0"/>
        **config_kwargs (additional keyword arguments):
            Keyword arguments to be passed to the [`BuilderConfig`]
            and used in the [`DatasetBuilder`].

    Returns:
        [`DatasetBuilder`]

    Example:

    ```py
    >>> from datasets import load_dataset_builder
    >>> ds_builder = load_dataset_builder('rotten_tomatoes')
    >>> ds_builder.info.features
    {'label': ClassLabel(num_classes=2, names=['neg', 'pos'], id=None),
     'text': Value(dtype='string', id=None)}
    ```
    N)r  r   r   r>  r?  r?  rM  r   z.Please specify the data files to load for the z dataset builder.c                 6    g | ]}t           |         k    |S r   )r*   )r   	extensionrK   s     rQ   r   z(load_dataset_builder.<locals>.<listcomp>  s2     
 
 
#?ST]?^bf?f?fI?f?f?frf   z9
For example `data_files={"train": "path/to/data/train/*.r   z"}`)r  rM  r>  r?  r   r  r   r  r   )r   r  r   r   r   r  re   r^   r   popr,   r*   r   )rK   rC   r>  r?  r  r  r   r   r  r   r  config_kwargsdataset_modulebuilder_clsr   rM  r   	error_msgexample_extensionsbuilder_instances   `                   rQ   load_dataset_builderr  }  s   z !!V,2VWWM!4CY/..000IYIY)7&+'#  N $N$>??K#2N##L*==J $$]D99Kf%%D)))j.@\T\\\	
 
 
 
';
 
 
  	srWijkWlrrrrI### (3{ (%'( ( ( ( ( rf   r0   r   verification_mode
save_infostask	streamingnum_procc                 J   |
dk    r=|
rt           j        nt           j        }	t          j        d|	j         dt                     ||st          d| d          t          | t          j
                                                  rt          d          |r|t          d          t          |pt          j                  }t          |s|	pt           j        nt           j                  }	t!          d| ||||||||||d	|}|r|                    |
          S | t$          v}|                    |||	|||           ||nt)          |j        j                  }|                    ||	|          }||                    |          }|r|                                 |S )a'  Load a dataset from the Hugging Face Hub, or a local dataset.

    You can find the list of datasets on the [Hub](https://huggingface.co/datasets) or with [`datasets.list_datasets`].

    A dataset is a directory that contains:

    - some data files in generic formats (JSON, CSV, Parquet, text, etc.).
    - and optionally a dataset script, if it requires some code to read the data files. This is used to load any kind of formats or structures.

    Note that dataset scripts can also download and read data files from anywhere - in case your data files already exist online.

    This function does the following under the hood:

        1. Download and import in the library the dataset script from `path` if it's not already cached inside the library.

            If the dataset has no dataset script, then a generic dataset script is imported instead (JSON, CSV, Parquet, text, etc.)

            Dataset scripts are small python scripts that define dataset builders. They define the citation, info and format of the dataset,
            contain the path or URL to the original data files and the code to load examples from the original data files.

            You can find the complete list of datasets in the Datasets [Hub](https://huggingface.co/datasets).

        2. Run the dataset script which will:

            * Download the dataset file from the original URL (see the script) if it's not already available locally or cached.
            * Process and cache the dataset in typed Arrow tables for caching.

                Arrow table are arbitrarily long, typed tables which can store nested objects and be mapped to numpy/pandas/python generic types.
                They can be directly accessed from disk, loaded in RAM or even streamed over the web.

        3. Return a dataset built from the requested splits in `split` (default: all).

    It also allows to load a dataset from a local directory or a dataset repository on the Hugging Face Hub without dataset script.
    In this case, it automatically loads all the data files from the directory or the dataset repository.

    Args:

        path (`str`):
            Path or name of the dataset.
            Depending on `path`, the dataset builder that is used comes from a generic dataset script (JSON, CSV, Parquet, text etc.) or from the dataset script (a python file) inside the dataset directory.

            For local datasets:

            - if `path` is a local directory (containing data files only)
              -> load a generic dataset builder (csv, json, text etc.) based on the content of the directory
              e.g. `'./path/to/directory/with/my/csv/data'`.
            - if `path` is a local dataset script or a directory containing a local dataset script (if the script has the same name as the directory)
              -> load the dataset builder from the dataset script
              e.g. `'./dataset/squad'` or `'./dataset/squad/squad.py'`.

            For datasets on the Hugging Face Hub (list all available datasets and ids with [`datasets.list_datasets`])

            - if `path` is a dataset repository on the HF hub (containing data files only)
              -> load a generic dataset builder (csv, text etc.) based on the content of the repository
              e.g. `'username/dataset_name'`, a dataset repository on the HF hub containing your data files.
            - if `path` is a dataset repository on the HF hub with a dataset script (if the script has the same name as the directory)
              -> load the dataset builder from the dataset script in the dataset repository
              e.g. `glue`, `squad`, `'username/dataset_name'`, a dataset repository on the HF hub containing a dataset script `'dataset_name.py'`.

        name (`str`, *optional*):
            Defining the name of the dataset configuration.
        data_dir (`str`, *optional*):
            Defining the `data_dir` of the dataset configuration. If specified for the generic builders (csv, text etc.) or the Hub datasets and `data_files` is `None`,
            the behavior is equal to passing `os.path.join(data_dir, **)` as `data_files` to reference all the files in a directory.
        data_files (`str` or `Sequence` or `Mapping`, *optional*):
            Path(s) to source data file(s).
        split (`Split` or `str`):
            Which split of the data to load.
            If `None`, will return a `dict` with all splits (typically `datasets.Split.TRAIN` and `datasets.Split.TEST`).
            If given, will return a single Dataset.
            Splits can be combined and specified like in tensorflow-datasets.
        cache_dir (`str`, *optional*):
            Directory to read/write data. Defaults to `"~/.cache/huggingface/datasets"`.
        features (`Features`, *optional*):
            Set the features type to use for this dataset.
        download_config ([`DownloadConfig`], *optional*):
            Specific download configuration parameters.
        download_mode ([`DownloadMode`] or `str`, defaults to `REUSE_DATASET_IF_EXISTS`):
            Download/generate mode.
        verification_mode ([`VerificationMode`] or `str`, defaults to `BASIC_CHECKS`):
            Verification mode determining the checks to run on the downloaded/processed dataset information (checksums/size/splits/...).

            <Added version="2.9.1"/>
        ignore_verifications (`bool`, defaults to `False`):
            Ignore the verifications of the downloaded/processed dataset information (checksums/size/splits/...).

            <Deprecated version="2.9.1">

            `ignore_verifications` was deprecated in version 2.9.1 and will be removed in 3.0.0.
            Please use `verification_mode` instead.

            </Deprecated>
        keep_in_memory (`bool`, defaults to `None`):
            Whether to copy the dataset in-memory. If `None`, the dataset
            will not be copied in-memory unless explicitly enabled by setting `datasets.config.IN_MEMORY_MAX_SIZE` to
            nonzero. See more details in the [improve performance](../cache#improve-performance) section.
        save_infos (`bool`, defaults to `False`):
            Save the dataset information (checksums/size/splits/...).
        revision ([`Version`] or `str`, *optional*):
            Version of the dataset script to load.
            As datasets have their own git repository on the Datasets Hub, the default version "main" corresponds to their "main" branch.
            You can specify a different version than the default "main" by using a commit SHA or a git tag of the dataset repository.
        use_auth_token (`str` or `bool`, *optional*):
            Optional string or boolean to use as Bearer token for remote files on the Datasets Hub.
            If `True`, or not specified, will get token from `"~/.huggingface"`.
        task (`str`):
            The task to prepare the dataset for during training and evaluation. Casts the dataset's [`Features`] to standardized column names and types as detailed in `datasets.tasks`.
        streaming (`bool`, defaults to `False`):
            If set to `True`, don't download the data files. Instead, it streams the data progressively while
            iterating on the dataset. An [`IterableDataset`] or [`IterableDatasetDict`] is returned instead in this case.

            Note that streaming works for datasets that use data formats that support being iterated over like txt, csv, jsonl for example.
            Json files may be downloaded completely. Also streaming from remote zip or gzip files is supported but other compressed formats
            like rar and xz are not yet supported. The tgz format doesn't allow streaming.
        num_proc (`int`, *optional*, defaults to `None`):
            Number of processes when downloading and generating the dataset locally.
            Multiprocessing is disabled by default.

            <Added version="2.7.0"/>
        storage_options (`dict`, *optional*, defaults to `None`):
            **Experimental**. Key/value pairs to be passed on to the dataset file-system backend, if any.

            <Added version="2.11.0"/>
        **config_kwargs (additional keyword arguments):
            Keyword arguments to be passed to the `BuilderConfig`
            and used in the [`DatasetBuilder`].

    Returns:
        [`Dataset`] or [`DatasetDict`]:
        - if `split` is not `None`: the dataset requested,
        - if `split` is `None`, a [`~datasets.DatasetDict`] with each split.

        or [`IterableDataset`] or [`IterableDatasetDict`]: if `streaming=True`

        - if `split` is not `None`, the dataset is requested
        - if `split` is `None`, a [`~datasets.streaming.IterableDatasetDict`] with each split.

    Example:

    Load a dataset from the Hugging Face Hub:

    ```py
    >>> from datasets import load_dataset
    >>> ds = load_dataset('rotten_tomatoes', split='train')

    # Map data files to splits
    >>> data_files = {'train': 'train.csv', 'test': 'test.csv'}
    >>> ds = load_dataset('namespace/your_dataset_name', data_files=data_files)
    ```

    Load a local dataset:

    ```py
    # Load a CSV file
    >>> from datasets import load_dataset
    >>> ds = load_dataset('csv', data_files='path/to/local/my_dataset.csv')

    # Load a JSON file
    >>> from datasets import load_dataset
    >>> ds = load_dataset('json', data_files='path/to/local/my_dataset.json')

    # Load from a local loading script
    >>> from datasets import load_dataset
    >>> ds = load_dataset('path/to/local/loading_script/loading_script.py', split='train')
    ```

    Load an [`~datasets.IterableDataset`]:

    ```py
    >>> from datasets import load_dataset
    >>> ds = load_dataset('rotten_tomatoes', split='train', streaming=True)
    ```

    Load an image dataset with the `ImageFolder` dataset builder:

    ```py
    >>> from datasets import load_dataset
    >>> ds = load_dataset('imagefolder', data_dir='/path/to/images', split='train')
    ```
    r0   z'ignore_verifications' was deprecated in favor of 'verification_mode' in version 2.9.1 and will be removed in 3.0.0.
You can remove this warning by passing 'verification_mode=z
' instead.NzEmpty 'data_files': 'z3'. It should be either non-empty or None (default).zjYou are trying to load a dataset that was saved using `save_to_disk`. Please use `load_from_disk` instead.zLoading a streaming dataset in parallel with `num_proc` is not implemented. To parallelize streaming, you can wrap the dataset with a PyTorch DataLoader using `num_workers` > 1 instead.)rK   rC   r>  r?  r  r  r   r   r  r   r  r   )r   r   r  try_from_hf_gcsr  r  )r   r  	in_memoryr   )r<   	NO_CHECKS
ALL_CHECKSr  warnvaluer  r   r   r   DATASET_STATE_JSON_FILENAMErN   r  r   r  BASIC_CHECKSr  as_streaming_datasetr,   r  r=   rN  dataset_size
as_datasetprepare_for_task_save_infos)rK   rC   r>  r?  r   r  r  r   r   r  ignore_verificationsr  r  r  r   r  r  r  r  r  r  r  dss                          rQ   load_datasetr    sC   T |++:No,66TdTomIZI`m m m	
 	
 	

 jppppqqqD&455<<>> 
3
 
 	

  
X)!|
 
 	

 !!V,2VWWM(DNo		;.;TdTo 
 , '#%'      B4454AAA "<<O ))'#+'' *    )4:JK[K`Km:n:n  
	$	$5DUao	$	p	pB  && '$$&&&Irf   dataset_pathc                    |dk    r!t          j        dt                     |j        }t	          j        | |          }|d         }t          |          rt          |           }t          j	        }n't	          j
        d          }| }t          j        j	        }|                    |          st          d|  d          |                     ||t           j                            r@|                     ||t           j                            rt'          j        | ||          S |                     ||t           j                            rt-          j        | ||          S t          d|  d	          )
a  
    Loads a dataset that was previously saved using [`~Dataset.save_to_disk`] from a dataset directory, or
    from a filesystem using either [`~datasets.filesystems.S3FileSystem`] or any implementation of
    `fsspec.spec.AbstractFileSystem`.

    Args:
        dataset_path (`str`):
            Path (e.g. `"dataset/train"`) or remote URI (e.g.
            `"s3://my-bucket/dataset/train"`) of the [`Dataset`] or [`DatasetDict`] directory where the dataset will be
            loaded from.
        fs (`~filesystems.S3FileSystem` or `fsspec.spec.AbstractFileSystem`, *optional*):
            Instance of the remote filesystem used to download the files from.

            <Deprecated version="2.9.0">

            `fs` was deprecated in version 2.9.0 and will be removed in 3.0.0.
            Please use `storage_options` instead, e.g. `storage_options=fs.storage_options`.

            </Deprecated>

        keep_in_memory (`bool`, defaults to `None`):
            Whether to copy the dataset in-memory. If `None`, the dataset
            will not be copied in-memory unless explicitly enabled by setting `datasets.config.IN_MEMORY_MAX_SIZE` to
            nonzero. See more details in the [improve performance](../cache#improve-performance) section.

        storage_options (`dict`, *optional*):
            Key/value pairs to be passed on to the file-system backend, if any.

            <Added version="2.9.0"/>

    Returns:
        [`Dataset`] or [`DatasetDict`]:
        - If `dataset_path` is a path of a dataset directory: the dataset requested.
        - If `dataset_path` is a path of a dataset dict directory, a [`DatasetDict`] with each split.

    Example:

    ```py
    >>> from datasets import load_from_disk
    >>> ds = load_from_disk('path/to/dataset/directory')
    ```
    r0   z'fs' was deprecated in favor of 'storage_options' in version 2.9.0 and will be removed in 3.0.0.
You can remove this warning by passing 'storage_options=fs.storage_options' instead.)r  r   filez
Directory z
 not found)r  r  z@ is neither a `Dataset` directory nor a `DatasetDict` directory.)r  r  r  r  fsspecget_fs_token_pathsr%   r$   	posixpathrL   
filesystemrJ   rK   rN   r%  r   r   DATASET_INFO_FILENAMEr  r   load_from_diskDATASETDICT_JSON_FILENAMEr   )r	  fsr  r  fs_token_pathsdest_dataset_path	path_joins          rQ   r  r    s   Z 
\c	
 	
 	

 ,.|_]]]N$21$5B B !1,??N		v&&(GL	99&'' G E\ E E EFFF	yy,f.JKKLL 	
QSQZQZ	#V%GHHR R 	
 %l>crssss	99.0PQQ	R	R 
),~gvwwwwgggg
 
 	
rf   )Tr  )NNNNNNr(  )	Nr   r   NNFNNN)
NNNNNNNNNN)NNNNNNNNNr0   NFNNNFNN)r0   NN)r)  r   rU   rY   r   rJ   r  r   r  r  collectionsr   dataclassesr   pathlibr   typingr   r   r   r	   r
   r   r   r   r  r  huggingface_hubr   r!  r   arrow_datasetr   builderr   r?  r   r   r   r   r   r   r   r   r   dataset_dictr   r   download.download_configr   download.download_managerr   #download.streaming_download_managerr    r!   r"   r  r#   filesystemsr$   r%   rN  r&   r'   iterable_datasetr(   r  r)   packaged_modulesr*   r+   r,   r-   splitsr.   tasksr/   utils.deprecation_utilsr0   utils.file_utilsr1   r2   r3   r4   r5   r6   r7   r8   r9   utils.filelockr:   	utils.hubr;   utils.info_utilsr<   r=   utils.loggingr>   utils.metadatar?   utils.py_utilsr@   utils.versionrA   r   r   rn   r   rQ  MODULE_NAME_FOR_DYNAMIC_MODULESr   rR   re   rv   r~   r   r   r   boolr   r   r   r  r  r  r  r+  r3  r=  rf  rm  r}  r  r  r  r  intr  r  r  r   r  r   rf   rQ   <module>r3     sR             				              ! ! ! ! ! !       N N N N N N N N N N N N N N N N N N N N   ! ! ! ! ! !       " " " " " " # # # # # #
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 ; : : : : : : : 4 4 4 4 4 4 3 3 3 3 3 3 W W W W W W W W W W       D D D D D D D D / / / / / / / / - - - - - -                              / / / / / /
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 % $ $ $ $ $ ! ! ! ! ! ! @ @ @ @ @ @ @ @ % % % % % % + + + + + + ' ' ' ' ' ' " " " " " " 
H		72799::eWD  6gk   
 JRSXY]_bYbScJd       $ HU4CWY]^dYeCe=f4g    4%d3i %C % % % %(c #    >
>>(-c3S.@(A>T\]kTl>	%S/> > > >BS%
S%"S% S% 	S%
 c3h(S% 5c?+S% E,"345S% 	S% S% S% S%lc3h( 5c?+ 	
     38_   < RV "4<U49=M4NeCHo   :"""4<U49=M4N"eCHo" " " "J                
" " " " " " " "
" " " " " " " "
H/ H/ H/ H/ H/ 4 H/ H/ H/V./ ./ ./ ./ ./3 ./ ./ ./b.@ .@ .@ .@ .@*? .@ .@ .@bM@ M@ M@ M@ M@-B M@ M@ M@`.@ .@ .@ .@ .@#8 .@ .@ .@bo@ o@ o@ o@ o@+@ o@ o@ o@dX@ X@ X@ X@ X@(= X@ X@ X@v6@ 6@ 6@ 6@ 6@!6 6@ 6@ 6@r./ ./ ./ ./ ./ 4 ./ ./ ./f /3048<*."BFp
 p

p
uS'\*+p
 n-p
 E,"345	p

 #3-p
 smp
 tT3=>?p
 p
 p
 p
 p
f ]^^ /3048<*._u _u
_uuS'\*+_u n-_u E,"345	_u
 #3-_u _u _u _u _^_uD stt "&##' 048<.2L L
L#L L 	L
 }L C=L L n-L E,"345L uS'\*+L L L L utLb "_c##'048<.215&*H H
H
3-H smH sHSM73c8TW=FX@Y;Y3ZZ[\	H
 }H x H n-H E,"345H uS'\*+H U49-.H d^H H H H HZ "_c)-##'048<@D%%).215/3"&*'Q Q
Q
3-Q smQ sHSM73c8TW=FX@Y;Y3ZZ[\	Q
 E#u*%&Q }Q x Q n-Q E,"345Q  &6&; <=Q TNQ Q uS'\*+Q U49-.Q  5l*+
,!Q" #Q$ sm%Q& d^'Q* ;!4oEF+Q Q Q Qj rvL
 L
L
8@L
`him`nL

7K L
 L
 L
 L
 L
 L
rf   