
    +gdt                     r    d dl mZ d dlmZ d dlZd dlmZ d dlmZ ddlm	Z	 ddl
mZ  G d	 d
e          ZdS )    )PurePosixPath)OptionalN)AbstractFileSystem)DatasetInfo   )"get_authentication_headers_for_url)
hf_hub_urlc                   z     e Zd ZdZdZdZ	 	 ddee         dee         f fdZ	d Z
	 dd
edefdZd ZddZ xZS )HfFileSystemz/Interface to files in a Hugging face repository z	hf-legacyN	repo_infotokenc                 f     t                      j        | fi | || _        || _        d| _        dS )a  
        The file system can be instantiated using a huggingface_hub.hf_api.DatasetInfo object,
        and can be used to list and open files from a Hugging Face dataset repository with fsspec.

        Args:
            repo_info (:obj:``DatasetInfo``, `optional`):
                Dataset repository info from huggingface_hub.HfApi().dataset_info(...)
            token (:obj:``str``, `optional`):
                Hugging Face token. Will default to the locally saved token if not provided.
        N)super__init__r   r   	dir_cache)selfr   r   kwargs	__class__s       Alib/python3.11/site-packages/datasets/filesystems/hffilesystem.pyr   zHfFileSystem.__init__   s=      	((((("
    c           	         | j         ~i | _         | j        j        D ]l}|j        d dd| j         |j        <   | j                             d t          t          |j                  j                  d d         D                        kd S d S )Nfilenamesizetypec                 P    i | ]#}t          |          t          |          d dd$S )N	directoryr   )str).0ds     r   
<dictcomp>z*HfFileSystem._get_dirs.<locals>.<dictcomp>2   s@        AQ{ S S  r   )r   r   siblings	rfilenameupdatelistr   parents)r   hf_files     r   	_get_dirszHfFileSystem._get_dirs'   s    >!DN>2   $- "5 5w01
 %% !%mG4E&F&F&N!O!OPSQSPS!T      "! r   rbpathmodec                 0   t          | j        t                    st          d| j                   t	          | j        j        || j        j                  }t          j        ||t          || j
                  ddi                                          S )Nz;Open is only implemented for dataset repositories, but got )revision)use_auth_token	trust_envT)r.   headersclient_kwargs)
isinstancer   r   NotImplementedErrorr	   idshafsspecopenr   r   )r   r-   r.   r   urls        r   _openzHfFileSystem._open8   s     $.+66 	v%&tdhdr&t&tuuu*D4>;MNNN{6s4:VVV&-	
 
 

 $&&	r   c                     |                                   |                     |          }|| j        v r| j        |         S t          |          )N)r+   _strip_protocolr   FileNotFoundError)r   r-   r   s      r   infozHfFileSystem.infoH   sM    ##D))4>!!>$''#D)))r   Fc                    |                                   t          |                    d                    }i }| j                                        D ]F\  }}t          |                    d                    }|j        }||k    r||t          |          <   Gt          |                                          }|r|S t          d |D                       S )N/c              3   &   K   | ]}|d          V  dS )r   N )r!   fs     r   	<genexpr>z"HfFileSystem.ls.<locals>.<genexpr>]   s&      11!F)111111r   )
r+   r   stripr   itemsparentr    r(   valuessorted)	r   r-   detailr   pathsprE   rootouts	            r   lszHfFileSystem.lsP   s    TZZ__--N((** 	" 	"DAqaggcll++A8Dt|| !c!ff5<<>>"" 	2J11S111111r   )NN)r,   )F)__name__
__module____qualname____doc__root_markerprotocolr   r   r    r   r+   r<   r@   rQ   __classcell__)r   s   @r   r   r      s        99KH ,0# K( }     *  (       * * *2 2 2 2 2 2 2 2r   r   )pathlibr   typingr   r9   r   huggingface_hub.hf_apir   utils.file_utilsr   	utils.hubr	   r   rD   r   r   <module>r^      s    ! ! ! ! ! !        % % % % % % . . . . . . A A A A A A " " " " " "Q2 Q2 Q2 Q2 Q2% Q2 Q2 Q2 Q2 Q2r   