
    +gd                         d dl Z d dlZd dlmZmZmZ d dlZddlmZm	Z	m
Z
mZ ddlmZ ddlmZ ddlmZ ddlmZmZ d	d
lmZ  G d de          Z G d d          ZdS )    N)BinaryIOOptionalUnion   )DatasetFeatures
NamedSplitconfig)query_table)Json)logging)NestedDataStructureLikePathLike   )AbstractDatasetReaderc                        e Zd Z	 	 	 	 	 	 	 ddee         dee         dee         dede	de	d	ee         d
ee
         f fdZd Z xZS )JsonDatasetReaderNFpath_or_pathssplitfeatures	cache_dirkeep_in_memory	streamingfieldnum_procc	           
           t                      j        |f||||||d|	 || _        t          |t                    r|n| j        |i}t          d||||d|	| _        d S )N)r   r   r   r   r   r   )r   
data_filesr   r    )super__init__r   
isinstancedictr   r   builder)selfr   r   r   r   r   r   r   r   kwargs	__class__s             0lib/python3.11/site-packages/datasets/io/json.pyr    zJsonDatasetReader.__init__   s     			
)		
 		
 		
 		
 		
 
)3M4)H)Hitz[hNi 
$	
 

 
 
    c                     | j         r!| j                            | j                  }nSd }d }d }d }| j                            ||||| j                   | j                            | j        || j                  }|S )N)r   )download_configdownload_modeverification_mode	base_pathr   )r   r,   	in_memory)r   r#   as_streaming_datasetr   download_and_preparer   
as_datasetr   )r$   datasetr*   r+   r,   r-   s         r'   readzJsonDatasetReader.read0   s    > 	l77dj7IIGG #O M $IL-- /+"3# .    l--j4EQUQd .  G r(   )NNNFFNN)__name__
__module____qualname__r   r   r   r	   r   strboolintr    r3   __classcell__)r&   s   @r'   r   r      s         '+'+$#"&
 
.x8
 
#
 8$	

 
 
 
 }
 3-
 
 
 
 
 
@      r(   r   c            
       v    e Zd Z	 	 ddedeeef         dee         dee         fdZ	defdZ
d	 Zd
edefdZdS )JsonDatasetWriterNr2   path_or_buf
batch_sizer   c                     ||dk    rt          d| d          || _        || _        |r|nt          j        | _        || _        d| _        || _        d S )Nr   z	num_proc z must be an integer > 0.zutf-8)	
ValueErrorr2   r=   r
   DEFAULT_MAX_BATCH_SIZEr>   r   encodingto_json_kwargs)r$   r2   r=   r>   r   rC   s         r'   r    zJsonDatasetWriter.__init__J   sj     HMMKKKKLLL&(2U**8U ,r(   returnc           	         | j                             dd           }| j                             dd          }| j                             d|dk    rdnd          }| j                             d|dv rdnd          }| j                             d	d           }|d
vrt          d| d          t          | j        t
          t          t          j        f          rLt          j
        | j        d|          5 } | j        d||||d| j         }d d d            n# 1 swxY w Y   n1|rt          d| d           | j        d| j        |||d| j         }|S )Nr=   orientrecordslinesTFindex)r   tablecompression)Ninfergzipbz2xzz&`datasets` currently does not support z compressionwb)rK   )file_objrF   rH   rI   zUThe compression parameter is not supported when writing to a buffer, but compression=z1 was passed. Please provide a local path instead.r   )rC   popNotImplementedErrorr!   r=   r7   bytesosr   fsspecopen_write)r$   _rF   rH   rI   rK   bufferwrittens           r'   writezJsonDatasetWriter.write\   s   ##M488$((9==#''99L9LRWXX#''&DV:V:V\`aa)--mTBBBBB%&h{&h&h&hiiid&eR[(ABB 	T-tMMM wQW%$+vvfEY^vvbfbuvvw w w w w w w w w w w w w w w  )Hlw H H H   "dk )&U VZVi G s   5DD Dc                 :   |\  }}}}}t          | j        j        t          ||| j        z             | j        j                  } |                                j        dd |||d|}|                    d          s|dz  }|	                    | j
                  S )N)rJ   keyindices)r=   rF   rH   rI   
r   )r   r2   dataslicer>   _indices	to_pandasto_jsonendswithencoderB   )	r$   argsoffsetrF   rH   rI   rC   batchjson_strs	            r'   _batch_jsonzJsonDatasetWriter._batch_jsont   s    7;4un,#fft677L)
 
 

 -5??$$, 
V5
 
IW
 
   && 	Ht}---r(   rQ   c                    d}| j         | j         dk    rt          j        t          dt	          | j                  | j                  dt          j                     d          D ]4}|                     |f          }||	                    |          z  }5nt	          | j                  | j        }
}	t          j        | j                   5 }t          j        |                    | j        fdt          d|	|
          D                       |	|
z  r|	|
z  dz   n|	|
z  dt          j                     d          D ]}||	                    |          z  }	 ddd           n# 1 swxY w Y   |S )	zWrites the pyarrow table as JSON lines to a binary file handle.

        Caller is responsible for opening and closing the handle.
        r   Nr   bazCreating json from Arrow format)unitdisabledescc                     g | ]	}|f
S r   r   ).0ri   rI   rH   rF   rC   s     r'   
<listcomp>z,JsonDatasetWriter._write.<locals>.<listcomp>   s$    uuuF&&%Guuur(   )totalro   rp   rq   )r   r   tqdmrangelenr2   r>   is_progress_bar_enabledrl   r\   multiprocessingPoolimap)r$   rQ   rF   rH   rI   rC   r[   ri   rk   num_rowsr>   pools     ````      r'   rX   zJsonDatasetWriter._write   s    = DMQ$6$6!,aT\**DO<<#;===6	   4 4  ++VVUE>,Z[[8>>(3334 $'t|#4#4dojH %dm44 8 'II(uuuuuuuV[\]_gisVtVtuuu  ;CZ:Ok8z1Q66U]akUk ' ? A AA:	! 	! 	! 
8 
8H x~~h777GG
88 8 8 8 8 8 8 8 8 8 8 8 8 8 8 s   BE..E25E2)NN)r4   r5   r6   r   r   r   r   r   r9   r    r\   rl   rX   r   r(   r'   r<   r<   I   s        
 %)"&- -- 8X-.- SM	-
 3-- - - -$s    0. . .&& 
& & & & & &r(   r<   )rz   rU   typingr   r   r   rV    r   r   r	   r
   
formattingr   packaged_modules.json.jsonr   utilsr   utils.typingr   r   abcr   r   r<   r   r(   r'   <module>r      s-       				 , , , , , , , , , ,  4 4 4 4 4 4 4 4 4 4 4 4 $ $ $ $ $ $ - - - - - -       < < < < < < < < & & & & & &7 7 7 7 7- 7 7 7t` ` ` ` ` ` ` ` ` `r(   