
    +gdT                        d dl mZ d dlmZ d dlmZmZ d dlZ G d dej                  Z	de
deee
         e
f         fd	Z G d
 de          Zi dg dg dg dg dg dg dg dg dg dg dg dg dg dg dg dg dg i dg dg dg d g d!g d"g d#g d$g d%g d&g d'g d(g d)g d*g d+g d,g d-g g g g d.Zed/k    rd d0lmZ  ed12          Ze                    d3           e                                Z eej                  Ze                    e          Z ee           e                    e           dS dS )4    )Counter)Path)OptionalTupleNc                   &     e Zd Zd Zd fd	Z xZS )_NoDuplicateSafeLoaderc                       fd|j         D             }d |D             }t          |          fdD             }|rt          d|           d S )Nc                 0    g | ]\  }}j         |         S  )constructed_objects).0key_node_selfs      7lib/python3.11/site-packages/datasets/utils/metadata.py
<listcomp>zS_NoDuplicateSafeLoader._check_no_duplicates_on_constructed_node.<locals>.<listcomp>
   s%    QQQ{x(2QQQ    c                 Z    g | ](}t          |t                    rt          |          n|)S r   )
isinstancelisttuple)r   keys     r   r   zS_NoDuplicateSafeLoader._check_no_duplicates_on_constructed_node.<locals>.<listcomp>   s1    MMMjd33<c


MMMr   c                 ,    g | ]}|         d k    |S )   r   )r   r   counters     r   r   zS_NoDuplicateSafeLoader._check_no_duplicates_on_constructed_node.<locals>.<listcomp>   s'    EEE#GCL14D4D#4D4D4Dr   zGot duplicate yaml keys: )valuer   	TypeError)r   nodekeysduplicate_keysr   s   `   @r   (_check_no_duplicates_on_constructed_nodez?_NoDuplicateSafeLoader._check_no_duplicates_on_constructed_node	   s    QQQQdjQQQMMMMM$--EEEEEEE 	JHHHIII	J 	Jr   Fc                 x    t                                          ||          }|                     |           |S )N)deep)superconstruct_mappingr!   )r   r   r#   mapping	__class__s       r   r%   z(_NoDuplicateSafeLoader.construct_mapping   s8    ''++Dt+<<55d;;;r   )F)__name__
__module____qualname__r!   r%   __classcell__)r'   s   @r   r   r      sO        J J J         r   r   readme_contentreturnc                 d   t          |                                           }|rw|d         dk    rkd|dd          v r_|dd                              d          dz   }d                    |d|                   }|d                    ||dz   d                    fS d d                    |          fS )Nr   z---r   
)r   
splitlinesindexjoin)r,   full_contentsep_idx	yamlblocks       r   _split_yaml_from_readmer6      s    113344L AQ500Ul122>N5N5Nqrr"((//!3IIl1W9566	$))L1$?@@@@<((((r   c                       e Zd ZdhZededd fd            ZdefdZddee	         de	fdZ
ed	e	dd fd
            Zde	fdZdS )DatasetMetadatatrain_eval_indexpathr-   c                     t          |d          5 }t          |                                          \  }}ddd           n# 1 swxY w Y   ||                     |          S  |             S )aR  Loads and validates the dataset metadat from its dataset card (README.md)

        Args:
            path (:obj:`Path`): Path to the dataset card (its README.md file)

        Returns:
            :class:`DatasetMetadata`: The dataset's metadata

        Raises:
            :obj:`TypeError`: If the dataset's metadata is invalid
        utf-8encodingN)openr6   readfrom_yaml_string)clsr:   readme_fileyaml_stringr   s        r   from_readmezDatasetMetadata.from_readme%   s     $))) 	I[4[5E5E5G5GHHNK	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I"''444355Ls   %AA
Ac                 X   |                                 r>t          |d          5 }|                                }d d d            n# 1 swxY w Y   nd }|                     |          }t          |dd          5 }|                    |           d d d            d S # 1 swxY w Y   d S )Nr<   r=   w)existsr?   r@   
_to_readmewrite)r   r:   rC   r,   updated_readme_contents        r   	to_readmezDatasetMetadata.to_readme9   s-   ;;== 	"dW--- 4!,!1!1!3!34 4 4 4 4 4 4 4 4 4 4 4 4 4 4 "N!%!@!@$g... 	6+4555	6 	6 	6 	6 	6 	6 	6 	6 	6 	6 	6 	6 	6 	6 	6 	6 	6 	6s#   AAA<BB#&B#Nr,   c                     |0t          |          \  }}d|                                 z   dz   |z   }nd|                                 z   dz   }|S )Nz---
)r6   to_yaml_string)r   r,   r   contentr3   s        r   rI   zDatasetMetadata._to_readmeC   s\    %0@@JAw"T%8%8%:%::WDwNLL"T%8%8%:%::WDLr   stringc                      t          j        |t                    pi } fd|                                D             }  di |S )a'  Loads and validates the dataset metadata from a YAML string

        Args:
            string (:obj:`str`): The YAML string

        Returns:
            :class:`DatasetMetadata`: The dataset's metadata

        Raises:
            :obj:`TypeError`: If the dataset's metadata is invalid
        )Loaderc                     i | ];\  }}|                     d d          j        v r|                     d d          n||<S )-r   )replace_FIELDS_WITH_DASHES)r   r   r   rB   s      r   
<dictcomp>z4DatasetMetadata.from_yaml_string.<locals>.<dictcomp>[   s_     
 
 
U '*kk#s&;&;s?V&V&VS[[c"""\_bg
 
 
r   r   )yamlloadr   items)rB   rP   metadata_dicts   `  r   rA   z DatasetMetadata.from_yaml_stringK   sk     	&1GHHHNB
 
 
 
+1133
 
 
 s##]###r   c                      t          j         fd                                 D             ddd                              d          S )Nc                 Z    i | ]'\  }}|j         v r|                    d d          n||(S )r   rT   )rV   rU   )r   r   r   r   s      r   rW   z2DatasetMetadata.to_yaml_string.<locals>.<dictcomp>c   sP       C +.1I*I*IS#&&&sUZ  r   FTr<   )	sort_keysallow_unicoder>   )rX   	safe_dumprZ   decode)r   s   `r   rN   zDatasetMetadata.to_yaml_stringa   sb    ~   "&**,,   
 
 
 &//	r   )N)r(   r)   r*   rV   classmethodr   rE   rL   r   strrI   rA   rN   r   r   r   r8   r8   !   s        -.t (9    [&6d 6 6 6 6 # #     $c $.? $ $ $ [$*	 	 	 	 	 	 	r   r8   zimage-classificationtranslationzimage-segmentationz	fill-maskzautomatic-speech-recognitionztoken-classificationzsentence-similarityzaudio-classificationzquestion-answeringsummarizationzzero-shot-classificationztable-to-textzfeature-extractionotherzmultiple-choiceztext-classificationztext-to-imageztext2text-generationzzero-shot-image-classificationztabular-classificationztabular-regressionzimage-to-imageztabular-to-textzunconditional-image-generationztext-retrievalztext-to-speechzobject-detectionzaudio-to-audioztext-generationconversationalztable-question-answeringzvisual-question-answeringzimage-to-textzreinforcement-learning)zvoice-activity-detectionztime-series-forecastingzdocument-question-answering__main__)ArgumentParserz5Validate the yaml metadata block of a README.md file.)usagereadme_filepath)collectionsr   pathlibr   typingr   r   rX   
SafeLoaderr   rc   r6   dictr8   known_task_idsr(   argparseri   apadd_argument
parse_argsargsrk   rE   dataset_metadataprintrL   r   r   r   <module>ry      sL               " " " " " " " "     T_   )C )E(3-:L4M ) ) ) )I I I I Id I I I^&B&2& "& 	&
 #B& B& 2& B& "& R& & R& "& R& r&  2!&" R#& &$ B%&& %b'&( b)&* "+&, b-&. r/&0 %b1&2 b3&4 b5&6 7&8 b9&: r;&< b=&> ?&@  A&B RC&D bE& &F !#!#%K& & &R z''''''	U	V	V	VBOO%&&&==??Dd4/00O&22?CC	E
///// r   