
    +gd:!                         d dl Z d dlmZ d dlmZ d dlmZmZ d dlm	Z	 d dl
Zd dlmZ d dlmZ d dlmZ d d	lmZmZ d d
lmZ d dlmZ d dlmZmZ  ee          Zd Z G d de          ZdS )    N)ArgumentParser)Path)copyfilermtree)	Generator)DatasetBuilder)BaseDatasetsCLICommand)DownloadMode)dataset_module_factoryimport_main_class)logger)VerificationMode)ERROR
get_loggerc                     t          | j        | j        | j        | j        | j        | j        p| j        | j        | j	        | j
        	  	        S )N)TestCommanddatasetname	cache_dirdata_dirall_configs	save_info
save_infosignore_verificationsforce_redownloadclear_cache)argss    6lib/python3.11/site-packages/datasets/commands/test.py_test_command_factoryr      sL    	)$/!
 
 
    c                   `    e Zd ZdZedefd            Zdededededed	ed
ededefdZ	d Z
dS )r   Fparserc                 x   |                      dd          }|                    dt          d d           |                    dt          d d           |                    d	t          d d
           |                    ddd           |                    ddd           |                    ddd           |                    ddd           |                    ddd           |                    ddd           |                    dt          d           |                    t                     d S )NtestzTest dataset implementation.)helpz--namezDataset processing name)typedefaultr%   z--cache_dirz.Cache directory where the datasets are stored.z
--data_dirz@Can be used to specify a manual directory to get the files from.z--all_configs
store_truezTest all dataset configurations)actionr%   z--save_infoz6Save the dataset infos in the dataset card (README.md)z--ignore_verificationsz1Run the test without checksums and splits checks.z--force_redownloadzForce dataset redownloadz--clear_cachezBRemove downloaded files and cached datasets after each config testz--save_infoszalias to save_infor   zName of the dataset to download)r&   r%   )func)
add_parseradd_argumentstrset_defaultsr   )r"   test_parsers     r   register_subcommandzTestCommand.register_subcommand%   s   ''5S'TT  THa bbb  A	 	! 	
 	
 	
 	  S	 	! 	
 	
 	
 	  Lm nnn  ,5m 	! 	
 	
 	
 	  $D 	! 	
 	
 	

 	  !5lQk lll  U 	! 	
 	
 	
 	  K_ ```  ;\ ]]]  &; <<<<<r    r   r   r   r   r   r   r   r   r   c
                     || _         || _        || _        || _        || _        || _        || _        || _        |	| _        |	r |st          d           t          d           |r	d| _        d S d S )Na  When --clear_cache is used, specifying a cache directory is mandatory.
The 'download' folder of the cache directory and the dataset builder cache will be deleted after each configuration test.
Please provide a --cache_dir that will be used to test the dataset script.   T)_dataset_name
_cache_dir	_data_dir_all_configs_save_infos_ignore_verifications_force_redownload_clear_cacheprintexit)
selfr   r   r   r   r   r   r   r   r   s
             r   __init__zTestCommand.__init__I   s      
#!'%%9"!1' 	y 	]  
 GGG 	.)-D&&&	. 	.r    c           
      b    t                                          t                      j        % j        rt          d           t          d            j         j        c}t          |          t          j
                   j        rj        rt          j                  nd}dt          t          d d f         f fd}t           |                      D ]<\  }}t          d|j        j         d|dz    d| d           t$          j                            t$          j                            |                                t.          j        j                            |_        |                     j        st8          j        nt8          j         j        rt@          j!        nt@          j"        d	
           |#                                  j$        r|$                                  j$        rDt$          j                                                            d          }tK          |          j        dz   }t$          j                            ||          }t$          j        &                    |          r t$          j        '                    |          }	nXt$          j        &                    |          r|}	n6t$          j        (                    |          r|}	nd }	t          d|            |	Bt$          j                            |	d          }
tS          ||
           t          d|
             j*        rt$          j        (                    |j+                  r6tX          -                    d|j+                    t]          |j+                   t$          j                             j+        t.          j        j/                  }t$          j        (                    |          r,tX          -                    d|            t]          |           >t          d           d S )NzABoth parameters `config` and `all_configs` can't be used at once.r2   returnc               3   x  K   j         rkj        rdt          j                  D ]M\  } }dj        v r dj        j        dj        V  + d|j        j        j        dj        V  Nd S dj        v r dj        j        dj        V  d S  dj        j        dj        V  d S )Nconfig_name)r   r   )rC   r   r    )r7   BUILDER_CONFIGS	enumeratebuilder_kwargsr5   r6   r   )iconfigbuilder_clsrC   moduler>   s     r   get_buildersz%TestCommand.run.<locals>.get_buildersr   sn       [%@ !*;+F!G!G  IAv$(===)k &*o%)^  %3      *k (.&*o%)^  %3	       !F$999%+r$.rr\b\qrrrrrrr%+ $/"&/!%  !/	      r    zTesting builder 'z' (/)F)download_modeverification_modetry_from_hf_gcsz	README.mdz.pyzDataset card saved at zClearing cache at zTest successful.)0	fl_loggersetLevelr   r4   r7   r<   r=   r3   r   r   module_pathrE   lenr   r   rF   rI   r   ospathexistsjoinget_imported_module_dirdatasetsDATASETDICT_INFOS_FILENAME_record_infosdownload_and_preparer:   r
   REUSE_CACHE_IF_EXISTSFORCE_REDOWNLOADr9   r   	NO_CHECKS
ALL_CHECKS
as_datasetr8   r   isfiledirnameisdirr   r;   r5   r   warningr   DOWNLOADED_DATASETS_DIR)r>   rW   
n_buildersrL   jbuilderdataset_readme_pathr   combined_pathdataset_diruser_dataset_readme_pathdownload_dirrJ   rC   rK   s   `           @@@r   runzTestCommand.runh   s   U###:!d&7!UVVVGGG M4:k'--'(:;;9=9Jq{OjqS4555pq
	id(BC 	 	 	 	 	 	 	 	 	6 $LLNN33 3	) 3	)JAwSgn&9SSa!eSSjSSSTTT$&GNNW<<>>@jkk% %G! ((-3l@@!2-#1"2"<"<%0 % )        &##%%%  O&(gll;3V3V3X3XZe&f&f#Dzz. "T4 8 87>>$'' J"$'//$"7"7KKW^^M22 J"&KKW]]4(( J"&KK"&KH3FHHIII */1w||K/U/U,02JKKKM3KMMNNN   )7==!344 /NN#L8J#L#LMMM7-...!w||DOX_=dee7==.. )NN#F#F#FGGG<((( !!!!!r    N)__name__
__module____qualname____test__staticmethodr   r0   r-   boolr?   rq   rD   r    r   r   r   "   s        H!=N != != != \!=F.. . 	.
 . . . #. . . . . .>Z" Z" Z" Z" Z"r    r   ) rV   argparser   pathlibr   shutilr   r   typingr   datasets.configr[   datasets.builderr   datasets.commandsr	   "datasets.download.download_managerr
   datasets.loadr   r   datasets.utils.filelockr   rR   datasets.utils.info_utilsr   datasets.utils.loggingr   r   rr   r   r   rD   r    r   <module>r      sQ   				 # # # # # #       # # # # # # # #           + + + + + + 4 4 4 4 4 4 ; ; ; ; ; ; C C C C C C C C 7 7 7 7 7 7 6 6 6 6 6 6 4 4 4 4 4 4 4 4 
H		  `" `" `" `" `"( `" `" `" `" `"r    