+gd`dZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl m Z mZddlmZddlmZddlmZddlmZmZmZmZmZddlmZmZddlZddlZddl Z dd lm!Z!dd l"m#Z#d d l$m%Z%m&Z&d d l'm(Z(ddl$m)Z)ddl*m+Z+ddl,m-Z-e)j.e/Z0dZ1ede2eZ3dRdeeee2fde2fdZ4de2de5fdZ6de2de5fdZ7de2de5fdZ8de3de3fdZ9dSde2d e2de2fd!Z: dTde2d e2dee j;ed&e2d'e2de2fd(Z?d)e2de2fd*Z@dRd+ZA dRde2fd,ZBdRd-eee2eCfde2fd.ZDdRd/e2d0eee2e5fdeCfd1ZEGd2d3eFZGdRd4ee2fd5ZHdde jIjJddd6d fd7eeKd8eeCd9ee jIjJd:eeeLd;eLdZN dVd@e2d/e2d;eLd O $ $ +r 1 mRW__XoE^E^EehlEl5m5mmr8crt|jdkotj| S)Nr:)rr5r&r$isabsr<s r+is_relative_pathr@Ls. O $ $ +r 1 X"'--:X:X6XXr8r$c tjtjtjt |}t |trt|n|S)z'Convert relative path to absolute path.)r&r$abspath expanduser expandvarsr" isinstancer)r$ abs_path_strs r+relative_to_absolute_pathrGPs]7??27#5#5bg6H6HT6S6S#T#TUUL!+D$!7!7 I4   \Ir8FT identifierfilenamec|r|r tjn tj}n|r tjn tj}d|||fSN/)r"CLOUDFRONT_DATASETS_DISTRIB_PREFIXS3_DATASETS_BUCKET_PREFIX!CLOUDFRONT_METRICS_DISTRIB_PREFIXS3_METRICS_BUCKET_PREFIXr))rHrIuse_cdndatasetendpoints r+ hf_bucket_urlrTVsPl@Gm6<z#url_or_path_join..rsJ*t*t^f3x==+@+@+M+M+T+TUX+Y+Y*t*t*t*t*t*tr8)r7 posixpathr)ras_posix)rdres r+url_or_path_joinrqps\Y6~iu*t*tjs*t*t*tuuuuI* ***33555r8 url_or_pathct|r|d|dStj|SrK)r7rindexr&r$dirname)rrs r+url_or_path_parentrvwsE[!!,4[//44455w{+++r8c*|d}t|}|}|r>|d}t|}|d|zz }|dr|dz }|S)a Convert `url` into a hashed filename in a repeatable way. If `etag` is specified, append its hash to the url's, delimited by a period. If the url ends with .h5 (Keras HDF5 weights) adds '.h5' to the name so that TF 2.0 can identify it as a HDF5 file (see https://github.com/tensorflow/tensorflow/blob/00fad90125b18b80fe054de1055770cfb8fe4ba3/tensorflow/python/keras/engine/network.py#L1380) utf-8.z.py)encoder hexdigestendswith)urletag url_bytesurl_hashrI etag_bytes etag_hashs r+hash_url_to_filenamer~s 7##Ii  H!!##H 0[[)) :&& C)--//// ||EE Or8c | td i|}|jp tj}t |t rt |}t |t rt |}t|rTt|||j |j |j |j |j |j|j|j|j|j|j }nWt(j|r|}n5t/|rt1d|dt3d|d||S|jr/t7|j||j}|S) a Given something that might be a URL (or might be a local path), determine which. If it's a URL, download the file and cache it, and return the path to the cached file. If it's already a local path, make sure the file exists and then return the path. Return: Local path (string) Raises: FileNotFoundError: in case of non-recoverable file (non-existent or no cache on disk) ConnectionError: in case of unreachable url and no cache on disk ValueError: if it couldn't parse the url or filename correctly requests.exceptions.ConnectionError: in case of internet connection issue N) cache_dirforce_downloadproxiesresume_download user_agentlocal_files_onlyuse_etagrVuse_auth_tokenignore_url_paramsstorage_options download_descz Local file z doesn't existzunable to parse z as a URL or as a local path)r) force_extract)rrrDOWNLOADED_DATASETS_PATHrErr"r7get_from_cacherrrrrrrVrrrrr&r$r(r=FileNotFoundError ValueErrorextract_compressed_filerextractr)r-download_configdownload_kwargsr output_paths r+ cached_pathrs,(;;?;;)LV-LI)T""# NN /4((/o.._%%[$ *9#++;&1,=$-'3*9-?+;)7     ( ([%  ' '[ Mo M M MNNNYOYYYZZZ. $/HIIIQQ 'DR   r8rcJdt}|dtjz }|dtjz }|dtjz }tjr|dtjz }tjr|dtjz }tj r|dtj z }tj r|dtj z }t|tr8|d d d |Dz }nt|t"r|d |zz }|S) Nz datasets/z ; python/z; huggingface_hub/z ; pyarrow/z; torch/z ; tensorflow/z; jax/z; apache_beam/z; c3*K|]\}}|d|VdSrhr)rlkvs r+rnz*get_datasets_user_agent..s0GGDAqZZAZZGGGGGGr8)rr PY_VERSIONhuggingface_hubPYARROW_VERSIONTORCH_AVAILABLE TORCH_VERSION TF_AVAILABLE TF_VERSION JAX_AVAILABLE JAX_VERSIONBEAM_AVAILABLE BEAM_VERSIONrEdictr)itemsr")ruas r+get_datasets_user_agentrsI "[ " "B )f' ) ))B <: < <>>NE7 '   ''bv||~~3PWbb[abbHGG#2H4G4WX ' ' '{""  vee3eeQVWbQbeeefff uqyAQ0QRR  :&&&&&&&&  '  ' Os)A %C/ACCctd|tj||\}}}t|dkrt d|||dS)Nrrrz=HEAD can be called with at most one path but was called with r)rfsspecget_fs_token_pathslenrr)r}rfs_pathss r+ fsspec_headrLsq%&=&=&=>>>,S/RRRLB5 5zzA~~`Y^``aaa 7758  r8c\td|tj||\}}}t|dkrt d|tj|pdtj d}| |d|j | dS) Nrrrz>>,S/RRRLB5 5zzA~~_X]__```,,)M":<<<  -H KKa).8K<<<<>> V^++C+AA B B a FF1III                uu 4s5.A0A$ A0$A((A0+A(,A00 A>=A>ctd| td|d|jt t j||5}tj ||ddddS#1swxYwYdS#t j j $r}t|dd}~wwxYw)NrzGetting through FTP z into r) rrrrYrrrrshutil copyfileobjerrorURLErrorr)r}rrres r+ftp_getrls%&=&=&=>>>+ F3FFinFFGGG V^++C+AA B B -a  q) , , , - - - - - - - - - - - - - - - - - - < +++a  d*+s<AB'B = B BBBBC+B;;CY@c vtj|pi}t|d|d<|dkr d|dd|d<t d|d ||||| } | jd krdS| jd } | |t| znd} tj d d | ||pdtj 5} | dD]9} | t| || : ddddS#1swxYwYdS)N user-agentrrzbytes=d-RangeGETT)rr}streamrrcookiesrVrizContent-LengthBr)unit unit_scaletotalinitialrri) chunk_size)copydeepcopyrgetrrrintrtqdmr iter_contentupdaterwrite)r}rr resume_sizerrrrVrrcontent_lengthrprogresschunks r+http_getr#vsmG$$*G3w{{t@j!j"f$r }|}Yd}~nd}~wwxYw|st j |r|s|S|rtGd|d ||jd!krtGd"|tId#||#t=d$|d%tK|d&|t=d$|d'|jd&t=d$|t||}t j ||}t j |r|s|S|d(z}tM|5|rU|d)z$tN$fd*}|}t j $rt j($j)}n!d+}ntUtVj,|d,}d+}|5}t6|d-|j-|dkrt]||n/|d vrt_||| | .nta||||||| | /dddn #1swxYwYt6d0|d1|tcj2|j-|t j3d2} t j3| t j4|d2| zt6d3|||d4}!|d5z}"tk|"d6d785}#tmj7|!|#dddn #1swxYwYdddn #1swxYwY|S)9a Given a URL, look for the corresponding file in the local cache. If it's not there, download it. Then return the path to the cached file. Return: Local path (string) Raises: FileNotFoundError: in case of non-recoverable file (non-existent or no cache on disk) ConnectionError: in case of unreachable url and no cache on disk NTrF)r~r)r r4r(rr+r~)r&rrrVrdownload_warningzdrive.google.comz &confirm=zconfirm=z &confirm=tizfirebasestorage.googleapis.comiiz7^https?://github.com/.*?/.*?/releases/download/.*?/.*?$z#^https://.*?s3.*?amazonaws.com/.*?$zndownloader.figstatic.comz"Couldn't get ETag version for url izUnauthorized for URL z^. Please use the parameter `use_auth_token=True` after logging in with `huggingface-cli login`z6Cannot find the requested files in the cached path at zi and outgoing traffic has been disabled. To enable file online look-ups, set 'local_files_only' to False.izCouldn't find file at rzCouldn't reach z ()z (error z.lockrc3fKtd5}|VddddS#1swxYwYdS)Nza+b)r*)fincomplete_paths r+_resumable_file_managerz/get_from_cache.._resumable_file_managerJs/511QGGGs &**r)dirdeletezB not found in cache or force_download set to True, downloading to )rr)rrrrrVrzstoring z in cache at izcreating metadata file for )r}r~z.jsonr rx)encoding)8rHF_DATASETS_CACHErErr"r&r'r rr$rr)r(rr5rrrrWrrrrrrematchr}rrrrOSErrorrrTimeoutrrreprrrstatst_sizertempfileNamedTemporaryFilerYrrr#rmoveumaskchmodr*jsondump)%r}rrr etag_timeoutrrrrrVrrrr cached_url connectedrrr~ head_errorr5rI cache_pathrrrr lock_pathr7temp_file_managerrrrFmeta meta_path meta_filer6s% @r+rrs :, )T""# NN K D))))S(3--"455  IHG DJ F$JT:::Hi22J w~~j!!.1^TTTG *  3#% U??  II , , ,"3HHHHQYcHLL..L(,,vt2L2L_cDI*   $$' H#s**7?Ix'++F333T$,224433DAq||$6773>* % % n    D#^^^^  !h&:c&A&A#$BS$B$BCC C)*AC*A*ABBB  !!"NC"N"N4 ;K;K"N"N"NOO O  !!"XC"X"XAU"X"X"XYY Y!"9C"9"9:: :$J55Hi22J w~~j!!.W$I )  2'2'  (=8O     ^ !8 w~~o..  go66>  '(C[` a a a K   I KK3rrbkbprr s s sY''''0003 ?Q^_____# +## +&                   (  =s====>>> INJ///  UeV^,,, >*>>???D))( )S7 3 3 3 'y IdI & & & ' ' ' ' ' ' ' ' ' ' ' ' ' ' 'c2'2'2'2'2'2'2'2'2'2'2'2'2'2'2'h sv E J++KK  K;BW*;A+S2& W*2S6 6W*9S6 :B7W*1W W*W W*W W**W.1W.cfd}|S)Ncdddz|j|jndz|_|SNr:z )r)__doc__fndocstrs r+docstring_decoratorz1add_start_docstrings..docstring_decorator~s1WWV__v-rz?U[]^  r8rrZr[s` r+add_start_docstringsr]}$ r8cfd}|S)Ncd|j|jnddzdz|_|SrV)rWr)rXs r+r[z/add_end_docstrings..docstring_decorators1$&J$:bjjfLrwwW]^  r8rr\s` r+add_end_docstringsrar^r8c4td|DS)Nc3HK|]}|jVdSN)rArB)rlr$s r+rnz(estimate_dataset_size..s-55ttyy{{"555555r8)sum)rs r+estimate_dataset_sizerfs 55u555 5 55r8r5ct} |d}|sn||z }|drn4t|S)NTr ) bytearrayrr|bytes)r5resbs r+readlinerms] ++C FF1II   q <<      ::r8rd)FT)FTr)TN)rrrr)NN)r)NrNNrrN)NNNTrr) NFNr/FNFTrNFNN)]rWriorHr&ror<rr#rCrr contextlibrr functoolsrhashlibrpathlibrtypingrr r r r urllib.parser rrrrr packagingrr:rrdownload.download_configrrrrfilelockr get_loggerrrINCOMPLETE_SUFFIXr"rr,boolr7r=r@rGrTResponserrXrcrqrvrrrrrrrrrRequestExceptiontuplerfloatrrrrrrr#rWr.rr]rarf RawIOBasermrr8r+rs7    ........77777777777777******** $$$$$$""""""""555555######  H % %! GCdhuT3Y/?&@C&I3I4IIII n3n4nnnnYcYdYYYYJAJ!JJJJ 66c6S6RU6666NO" 8 i '(WWW3W WY\WWWW6666666,C,C,,,,4BB BBBBJsDy1A(Bc(  C %PSUYPYJZA[ gk          ?     8C=    "&"&=E=P=a(,$X(9:  49%   : $$ $ $$ $  $  $$$$$N = = = =++++qu####Fde $c8E#t)4D+EQYZ]Q^  FF FFFFR666       r8