
    l$eE                     ^   d Z ddlZddlZddlZddlZddlmZ ddlmZ ddl	m
Z
 ddlmZ ddlmZ ddlmZmZmZmZmZmZmZmZ dd	lmZ dd
lmZmZmZ ddlmZ ddl m!Z!m"Z"m#Z#m$Z$m%Z% ddl&m'Z'm(Z( erddl)m*Z*  e$j+        e,          Z- ej.        d          Z/dZ0dddZ1e G d d                      Z2e%	 d:dee2         dee3         de3de3dee3         deee4         ee4         f         fd            Z5 G d de          Z6 G d d e          Z7d!d"d#edee3         ddfd$Z8d%e4fd&Z9d'e4fd(Z:d)e4fd*Z;d!d"d+e3ddfd,Z<d!d"d-ed.e=d+e3ddf
d/Z>d-ed0e2d.e=dee3         fd1Z?d2ee         d3e3de7fd4Z@d!d"d5ee3         d.e=dee         fd6ZAd!d"d5ee3         d.e=dee         fd7ZB G d8 d9e          ZCdS );z.Git LFS related type definitions and utilities    N)AbstractContextManager)	dataclass)ceil)getsize)Path)TYPE_CHECKINGBinaryIODictIterableListOptionalTuple	TypedDict)HTTPBasicAuth)ENDPOINTHF_HUB_ENABLE_HF_TRANSFERREPO_TYPES_URL_PREFIXES)get_session   )get_token_to_sendhf_raise_for_statushttp_backoffloggingvalidate_hf_hub_args)sha256sha_fileobj)CommitOperationAddz^[0-9a-f]{40}$zlfs-multipart-uploadzapplication/vnd.git-lfs+json)AcceptzContent-Typec                       e Zd ZU dZeed<   eed<   eed<   edefd            Z	edefd            Z
ed	efd
            ZdS )
UploadInfoad  
    Dataclass holding required information to determine whether a blob
    should be uploaded to the hub using the LFS protocol or the regular protocol

    Args:
        sha256 (`bytes`):
            SHA256 hash of the blob
        size (`int`):
            Size in bytes of the blob
        sample (`bytes`):
            First 512 bytes of the blob
    r   sizesamplepathc                     t          |          }t          j        |d          5 }|                    d          d d         }t	          |          }d d d            n# 1 swxY w Y    | |||          S )Nrb   r!   r   r"   )r   ioopenpeekr   )clsr#   r!   filer"   shas         3lib/python3.11/site-packages/huggingface_hub/lfs.py	from_pathzUploadInfo.from_pathF   s    t}}WT4   	$DYYs^^DSD)Fd##C	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ sS8888s   -AA"%A"datac                     t          |                                          } | t          |          |d d         |          S )Nr&   )r!   r"   r   )r   digestlen)r+   r0   r-   s      r.   
from_byteszUploadInfo.from_bytesN   s>    Tll!!##sD		$tt*SAAAA    fileobjc                    |                     d          }|                    dt          j                   t	          |          }|                                }|                    dt          j                    | |||          S )Nr&   r   r'   )readseekr(   SEEK_SETr   tell)r+   r6   r"   r-   r!   s        r.   from_fileobjzUploadInfo.from_fileobjS   sp    c""Q$$$'""||~~Q$$$sS8888r5   N)__name__
__module____qualname____doc__bytes__annotations__intclassmethodstrr/   r4   r	   r<    r5   r.   r    r    3   s           MMM
IIIMMM9S 9 9 9 [9 Be B B B [B 98 9 9 9 [9 9 9r5   r    upload_infostoken	repo_typerepo_idendpointreturnc                    ||nt           }d}|t          v rt          |         }| d| | d}t                                          |t          dddgd | D             d	d
t          dt          |pd                              }t          |           |                                }|	                    dd          }	t          |	t                    st          d          d |	D             d |	D             fS )a  
    Requests the LFS batch endpoint to retrieve upload instructions

    Learn more: https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md

    Args:
        upload_infos (`Iterable` of `UploadInfo`):
            `UploadInfo` for the files that are being uploaded, typically obtained
            from `CommitOperationAdd.upload_info`
        repo_type (`str`):
            Type of the repo to upload to: `"model"`, `"dataset"` or `"space"`.
        repo_id (`str`):
            A namespace (user or an organization) and a repo name separated
            by a `/`.
        token (`str`, *optional*):
            An authentication token ( See https://huggingface.co/settings/tokens )

    Returns:
        `LfsBatchInfo`: 2-tuple:
            - First element is the list of upload instructions from the server
            - Second element is an list of errors, if any

    Raises:
        `ValueError`: If an argument is invalid or the server response is malformed

        `HTTPError`: If the server returned an error
    N /z.git/info/lfs/objects/batchuploadbasic	multipartc                 P    g | ]#}|j                                         |j        d $S )oidr!   )r   hexr!   ).0rP   s     r.   
<listcomp>z'post_lfs_batch_info.<locals>.<listcomp>   sG       
  "=,,.."K   r5   r   )	operation	transfersobjects	hash_algoaccess_tokenT)headersjsonauthr[   zMalformed response from serverc                 4    g | ]}d |vt          |          S error)_validate_batch_actionsrW   objs     r.   rX   z'post_lfs_batch_info.<locals>.<listcomp>   s+    OOO#G3<N<N	 	%	%<N<N<Nr5   c                 4    g | ]}d |v t          |          S rb   )_validate_batch_errorre   s     r.   rX   z'post_lfs_batch_info.<locals>.<listcomp>   s&    III'S..	s	#	#...r5   )r   r   r   postLFS_HEADERSr   r   r   r_   get
isinstancelist
ValueError)
rG   rH   rI   rJ   rK   
url_prefix	batch_urlresp
batch_infor[   s
             r.   post_lfs_batch_infors   ]   sF   F $/xxXHJ+++,Y7
NNjN'NNNI==!!;/ 
 +   "
 
 emt,,
 
   D( JnnY--Ggt$$ ;9::: 	POOOOIIwIII r5   c                   $    e Zd ZU eed<   eed<   dS )PayloadPartT
partNumberetagN)r=   r>   r?   rC   rB   rE   rF   r5   r.   ru   ru      s"         OOO
IIIIIr5   ru   c                   4    e Zd ZU dZeed<   ee         ed<   dS )CompletionPayloadTz?Payload that will be sent to the Hub when uploading multi-part.rU   partsN)r=   r>   r?   r@   rE   rB   r   ru   rF   r5   r.   ry   ry      s2         II	HHHr5   ry   rY   r   lfs_batch_actionc           	         t          |           |                    d          }|%t                              d| j         d           dS |d         d         }t          |           |d                             d          }|t          |           |                    di           }|                    d          }|S	 t          |          }n(# t          t          f$ r t          d	| d
          w xY wt          | |||d                    nt          | |d                    |t          |           t                                          |d         t          dt          |pd                    | j        j                                        | j        j        d          }t'          |           t                              | j         d           dS )a  
    Handles uploading a given object to the Hub with the LFS protocol.

    Can be a No-op if the content of the file is already present on the hub large file storage.

    Args:
        operation (`CommitOperationAdd`):
            The add operation triggering this upload.
        lfs_batch_action (`dict`):
            Upload instructions from the LFS batch endpoint for this object. See [`~utils.lfs.post_lfs_batch_info`] for
            more details.
        token (`str`, *optional*):
            A [user access token](https://hf.co/settings/tokens) to authenticate requests against the Hub

    Raises:
        - `ValueError` if `lfs_batch_action` is improperly formatted
        - `HTTPError` if the upload resulted in an error
    actionsNzContent of file z. is already present upstream - skipping uploadrP   verifyheader
chunk_sizezTMalformed response from LFS batch endpoint: `chunk_size` should be an integer. Got 'z'.href)rY   r   r   
upload_url)rY   r   USERT)usernamepasswordrT   )r`   r_   z: Upload successful)rd   rk   loggerdebugpath_in_repo_validate_lfs_actionrC   rn   	TypeError_upload_multi_part_upload_single_partr   ri   r   r   upload_infor   rV   r!   r   )	rY   r{   rH   r}   upload_actionverify_actionr   r   verify_resps	            r.   
lfs_uploadr      s	   ( ,---""9--Gn	(>nnnooo %Y/9M'''$Y/33H==M ]+++ x,,FL))J	ZJJI& 	 	 	ugquuu  	 	Yv*anouavwwwwwiM&<QRRRR  ]+++!mm((&!9J5=TX9Y9YZZZ".599;;YEZE_`` ) 
 

 	K(((
LLI*???@@@@@s   C %C8
lfs_actionc                     t          |                     d          t                    r=|                     d          7t          |                     d          t                    st	          d          | S ).validates response from the LFS batch endpointr   r   Nz"lfs_action is improperly formatted)rl   rk   rE   dictrn   )r   s    r.   r   r      se     	:>>&))3//?^^H%%-JNN8<T<TVZ1[1[-=>>>r5   lfs_batch_actionsc                    t          |                     d          t                    r(t          |                     d          t                    st	          d          |                     di                               d          }|                     di                               d          }|t          |           |t          |           | S )r   rU   r!   z)lfs_batch_actions is improperly formattedr}   rP   r~   )rl   rk   rE   rC   rn   r   )r   r   r   s      r.   rd   rd      s    (,,U33S99 FjIZI^I^_eIfIfhk>l>l FDEEE%)))R88<<XFFM%)))R88<<XFFM ]+++ ]+++r5   lfs_batch_errorc                    t          |                     d          t                    r(t          |                     d          t                    st	          d          |                     d          }t          |t
                    rPt          |                    d          t                    r(t          |                    d          t                    st	          d          | S )r   rU   r!   z'lfs_batch_error is improperly formattedrc   messagecode)rl   rk   rE   rC   rn   r   )r   
error_infos     r.   rh   rh     s    **511377 DJGZGZ[aGbGbdg<h<h DBCCC $$W--J:t$$Dz~~i00#66D z~~f--s33D
 BCCCr5   r   c                     |                      d          5 }t          d||          }t          |           ddd           dS # 1 swxY w Y   dS )aZ  
    Uploads `fileobj` as a single PUT HTTP request (basic LFS transfer protocol)

    Args:
        upload_url (`str`):
            The URL to PUT the file to.
        fileobj:
            The file-like object holding the data to upload.

    Returns: `requests.Response`

    Raises: `requests.HTTPError` if the upload resulted in an error
    T	with_tqdmPUTr0   N)as_filer   r   )rY   r   r6   responses       r.   r   r     s     
		T		*	* &gz@@@H%%%& & & & & & & & & & & & & & & & & &s   "AA
A
r   r   c                    t          || j        |          }t          }t          rJt          | j        t
                    s0t          | j        t                    st          j        d           d}|rt          | ||          nt          | ||          }t                                          |t          || j        j                                                  t                     }t#          |           dS )z@
    Uploads file using HF multipart LFS transfer protocol.
    )r   r   r   zlhf_transfer is enabled but does not support uploading from bytes or BinaryIO, falling back to regular uploadF)rY   sorted_parts_urlsr   )r_   r^   N)_get_sorted_parts_urlsr   r   rl   path_or_fileobjrE   r   warningswarn_upload_parts_hf_transfer_upload_parts_iterativelyr   ri   _get_completion_payloadr   rV   rj   r   )rY   r   r   r   r   use_hf_transferresponse_headerscompletion_ress           r.   r   r   '  s   
 /f)J_lvwww 0O!	 94c::	  94d;;	 
 		
 	
 	
   	x!IIZgqrrrr&N_lvwww  !]]''$%5y7L7S7W7W7Y7YZZ (  N
 '''''r5   r   c                     d t          d |                                 D             d           D             }t          |          }|t          |j        |z            k    rt          d          |S )Nc                     g | ]\  }}|S rF   rF   )rW   _r   s      r.   rX   z*_get_sorted_parts_urls.<locals>.<listcomp>K  s,     
 
 
Az 	
 
 
r5   c                     g | ]>\  }}|                                 r%t          |          d k    ,t          |d          |f?S )r   
   )isdigitr3   rC   )rW   part_numr   s      r.   rX   z*_get_sorted_parts_urls.<locals>.<listcomp>N  s\       (Hj##%% +.h--!*;*; Xr""J/*;*;*;r5   c                     | d         S Nr   rF   )ts    r.   <lambda>z(_get_sorted_parts_urls.<locals>.<lambda>S  s
    !A$ r5   )keyz0Invalid server response to upload large LFS file)sorteditemsr3   r   r!   rn   )r   r   r   sorted_part_upload_urls	num_partss        r.   r   r   J  s    
 
# ,2LLNN  
 
 
 

 
 
 +,,ID)J67777KLLL""r5   r   rU   c                     g }t          |           D ]U\  }}|                    d          }||dk    rt          d| d|dz              |                    |dz   |d           V||dS )Nrw   rN   zInvalid etag (`z`) returned for part r   )rv   rw   )rU   rz   )	enumeraterk   rn   append)r   rU   rz   part_numberr   rw   s         r.   r   r   \  s     "E()9:: 	
 	
Vzz&!!<42::[t[[+XY/[[\\\)Ao 	
 	
 	
 	
 '''r5   r   c                 b   g }|                      d          5 }t          |          D ]m\  }}t          |||z  |          5 }t          d||          }t	          |           |                    |j                   d d d            n# 1 swxY w Y   n	 d d d            n# 1 swxY w Y   |S )NTr   )	seek_from
read_limitr   r   )r   r   SliceFileObjr   r   r   r^   )	rY   r   r   r^   r6   part_idxpart_upload_urlfileobj_slicepart_upload_ress	            r.   r   r   k  sK    G			T		*	* 	8g)23D)E)E 	8 	8%Ho$x/%   8 ".uoM"Z"Z"Z#O44467778 8 8 8 8 8 8 8 8 8 8 8 8 8 8	8	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 Ns5   *B$<B?B$BB$BB$$B(+B(c                     	 ddl m} n# t          $ r t          d          w xY w	  || j        ||ddd          S # t
          $ r}t          d          |d }~ww xY w)	Nr   )multipart_uploadzFast uploading using 'hf_transfer' is enabled (HF_HUB_ENABLE_HF_TRANSFER=1) but 'hf_transfer' package is not available in your environment. Try `pip install hf_transfer`.         )	file_path
parts_urlsr   	max_filesparallel_failuresmax_retriesz~An error occurred while uploading using `hf_transfer`. Consider disabling HF_HUB_ENABLE_HF_TRANSFER for better error handling.)hf_transferr   ImportErrorrn   r   	ExceptionRuntimeError)rY   r   r   r   es        r.   r   r   |  s    
0000000 
 
 
Q
 
 	

/(!!
 
 
 	
    &
 
 	s   	 #= 
AAAc                   t    e Zd ZdZdededefdZd Zd Zdd	efd
Z	defdZ
ej        fdededefdZd ZdS )r   a  
    Utility context manager to read a *slice* of a seekable file-like object as a seekable, file-like object.

    This is NOT thread safe

    Inspired by stackoverflow.com/a/29838711/593036

    Credits to @julien-c

    Args:
        fileobj (`BinaryIO`):
            A file-like object to slice. MUST implement `tell()` and `seek()` (and `read()` of course).
            `fileobj` will be reset to its original position when exiting the context manager.
        seek_from (`int`):
            The start of the slice (offset from position 0 in bytes).
        read_limit (`int`):
            The maximum number of bytes to read from the slice.

    Attributes:
        previous_position (`int`):
            The previous position

    Examples:

    Reading 200 bytes with an offset of 128 bytes from a file (ie bytes 128 to 327):
    ```python
    >>> with open("path/to/file", "rb") as file:
    ...     with SliceFileObj(file, seek_from=128, read_limit=200) as fslice:
    ...         fslice.read(...)
    ```

    Reading a file in chunks of 512 bytes
    ```python
    >>> import os
    >>> chunk_size = 512
    >>> file_size = os.getsize("path/to/file")
    >>> with open("path/to/file", "rb") as file:
    ...     for chunk_idx in range(ceil(file_size / chunk_size)):
    ...         with SliceFileObj(file, seek_from=chunk_idx * chunk_size, read_limit=chunk_size) as fslice:
    ...             chunk = fslice.read(...)

    ```
    r6   r   r   c                 0    || _         || _        || _        d S N)r6   r   r   )selfr6   r   r   s       r.   __init__zSliceFileObj.__init__  s    "$r5   c                 $   | j                                         | _        | j                             dt          j                  }t          | j        || j        z
            | _	        | j                             | j        t          j                   | S r   )r6   r;   _previous_positionr9   osSEEK_ENDminr   r   _lenr(   r:   )r   end_of_streams     r.   	__enter__zSliceFileObj.__enter__  sl    "&,"3"3"5"5))!R[99)GHH	$."+666r5   c                 Z    | j                             | j        t          j                   d S r   )r6   r9   r   r(   r:   )r   exc_type	exc_value	tracebacks       r.   __exit__zSliceFileObj.__exit__  s%    $12;?????r5   nc                     |                                  }|| j        k    rdS | j        |z
  }| j                            |dk     r|nt	          ||                    }|S )Nr5   r   )r;   r   r6   r8   r   )r   r   posremaining_amountr0   s        r.   r8   zSliceFileObj.read  s`    iikk$)39s?|  QUU!1!1AGW@X@XYYr5   rL   c                 D    | j                                         | j        z
  S r   )r6   r;   r   r   s    r.   r;   zSliceFileObj.tell  s    |  ""T^33r5   offsetwhencec                    | j         }|| j        z   }|t          j        t          j        fv rE|t          j        k    r||z   n||z   }t          |t          ||                    }t          j        }na|t          j        k    r>| j        	                                }t          ||z
  t          |||z
                      }nt          d| d          | j                            ||          | j         z
  S )Nzwhence value z is not supported)r   r   r   r:   r   maxr   SEEK_CURr6   r;   rn   r9   )r   r   r   startendcur_poss         r.   r9   zSliceFileObj.seek  s    dibk2;///'-'<'<UV^^#,FFC 0 011F[FFr{""l''))G#fcGm*D*DEEFFFVFFFGGG|  004>AAr5   c              #   :   K   |                      d          V  d S )Ni  @ )r   )r8   r   s    r.   __iter__zSliceFileObj.__iter__  s&      ii/i*******r5   N)r   )r=   r>   r?   r@   r	   rC   r   r   r   r8   r;   r   r:   r9   r   rF   r5   r.   r   r     s        * *X% %S %c % % % %
  @ @ @ c    4c 4 4 4 4 /1k B B3 B Bc B B B B+ + + + +r5   r   r   )Dr@   r(   r   rer   
contextlibr   dataclassesr   mathr   os.pathr   pathlibr   typingr   r	   r
   r   r   r   r   r   requests.authr   huggingface_hub.constantsr   r   r   huggingface_hub.utilsr   utilsr   r   r   r   r   	utils.shar   r   _commit_apir   
get_loggerr=   r   compile	OID_REGEXLFS_MULTIPART_UPLOAD_COMMANDrj   r    rE   r   rs   ru   ry   r   r   rd   rh   r   rC   r   r   r   r   r   r   rF   r5   r.   <module>r     s   5 4 				 				 				  - - - - - - ! ! ! ! ! !                   \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ ' ' ' ' ' ' b b b b b b b b b b - - - - - - f f f f f f f f f f f f f f * * * * * * * *  0//////		H	%	%BJ())	5  -2  &9 &9 &9 &9 &9 &9 &9 &9R  #E E:&EC=E E 	E
 smE 4:tDz!"E E E EP    9   
       9A. 9A$ 9AxX[} 9Aae 9A 9A 9A 9AxT    t    4    &#7 &S &T & & & && ("6  (  (RU  (cf  (ko  (  (  (  (F#4 #j #c #VZ[^V_ # # # #$(d4j (s (GY ( ( ( (#8<S	OR	$Z   "#8<S	OR	$Z   8W+ W+ W+ W+ W+) W+ W+ W+ W+ W+r5   