
    +gd                         d dl Z d dlZd dlmZmZmZ ddlmZmZm	Z	m
Z
 ddlmZ ddlmZ ddlmZ ddlmZmZ d	d
lmZ  G d de          Z G d d          ZdS )    N)BinaryIOOptionalUnion   )DatasetFeatures
NamedSplitconfig)query_table)Csv)logging)NestedDataStructureLikePathLike   )AbstractDatasetReaderc                        e Zd Z	 	 	 	 	 	 ddee         dee         dee         dede	de	d	ee
         f fd
Zd Z xZS )CsvDatasetReaderNFpath_or_pathssplitfeatures	cache_dirkeep_in_memory	streamingnum_procc           
           t                      j        |f||||||d| t          |t                    r|n| j        |i}t          d|||d|| _        d S )N)r   r   r   r   r   r   )r   
data_filesr    )super__init__
isinstancedictr   r   builder)
selfr   r   r   r   r   r   r   kwargs	__class__s
            /lib/python3.11/site-packages/datasets/io/csv.pyr   zCsvDatasetReader.__init__   s     			
)		
 		
 		
 		
 		
 *4M4)H)Hitz[hNi 
$
 
 	
 
    c                     | j         r!| j                            | j                  }nSd }d }d }d }| j                            ||||| j                   | j                            | j        || j                  }|S )N)r   )download_configdownload_modeverification_mode	base_pathr   )r   r+   	in_memory)r   r"   as_streaming_datasetr   download_and_preparer   
as_datasetr   )r#   datasetr)   r*   r+   r,   s         r&   readzCsvDatasetReader.read+   s    > 	l77dj7IIGG #O M $IL-- /+"3# .    l--j4EQUQd .  G r'   )NNNFFN)__name__
__module____qualname__r   r   r   r	   r   strboolintr   r2   __classcell__)r%   s   @r&   r   r      s         '+'+$"&
 
.x8
 
#
 8$	

 
 
 
 3-
 
 
 
 
 
:      r'   r   c            
       v    e Zd Z	 	 ddedeeef         dee         dee         fdZ	defdZ
d	 Zd
edefdZdS )CsvDatasetWriterNr1   path_or_buf
batch_sizer   c                     ||dk    rt          d| d          || _        || _        |r|nt          j        | _        || _        d| _        || _        d S )Nr   z	num_proc z must be an integer > 0.zutf-8)	
ValueErrorr1   r<   r
   DEFAULT_MAX_BATCH_SIZEr=   r   encodingto_csv_kwargs)r#   r1   r<   r=   r   rB   s         r&   r   zCsvDatasetWriter.__init__E   sj     HMMKKKKLLL&(2U**8U *r'   returnc                    | j                             dd           }| j                             dd          }| j                             dd          }t          | j        t          t
          t          j        f          rDt          | j        d          5 } | j	        d|||d| j         }d d d            n# 1 swxY w Y   n | j	        d| j        ||d| j         }|S )	Nr<   headerTindexFzwb+)file_objrE   rF   r   )
rB   popr    r<   r6   bytesosr   open_write)r#   _rE   rF   bufferwrittens         r&   writezCsvDatasetWriter.writeW   s2   ""=$77#''$77"&&w66d&eR[(ABB 	od&.. i&%$+hvfEhhUYUghhi i i i i i i i i i i i i i i "dkn4+;FRWnn[_[mnnGs   B66B:=B:c                    |\  }}}}t          | j        j        t          ||| j        z             | j        j                  } |                                j        dd |dk    r|nd|d|}|                    | j	                  S )N)tablekeyindicesr   F)r<   rE   rF   r   )
r   r1   dataslicer=   _indices	to_pandasto_csvencoderA   )r#   argsoffsetrE   rF   rB   batchcsv_strs           r&   
_batch_csvzCsvDatasetWriter._batch_csvc   s    /3,},#fft677L)
 
 

 +%//##* 
!VV%u
 
Xe
 
 ~~dm,,,r'   rG   c                    d}| j         | j         dk    rt          j        t          dt	          | j                  | j                  dt          j                     d          D ]3}|                     |f          }||	                    |          z  }4nt	          | j                  | j        }	}t          j        | j                   5 }
t          j        |
                    | j        fdt          d||	          D                       ||	z  r||	z  dz   n||	z  dt          j                     d          D ]}||	                    |          z  }	 ddd           n# 1 swxY w Y   |S )	zWrites the pyarrow table as CSV to a binary file handle.

        Caller is responsible for opening and closing the handle.
        r   Nr   bazCreating CSV from Arrow format)unitdisabledescc                     g | ]}|f	S r   r   ).0r\   rE   rF   rB   s     r&   
<listcomp>z+CsvDatasetWriter._write.<locals>.<listcomp>   s"    mmmF&&%?mmmr'   )totalrb   rc   rd   )r   r   tqdmrangelenr1   r=   is_progress_bar_enabledr_   rP   multiprocessingPoolimap)r#   rG   rE   rF   rB   rO   r\   r^   num_rowsr=   pools     ```      r&   rL   zCsvDatasetWriter._writep   s   
 = DMQ$6$6!,aT\**DO<<#;===5	   3 3 //665-*PQQ8>>'2223 $'t|#4#4dojH %dm44 7&|IImmmmmmeTUW_akNlNlmmm  ;CZ:Ok8z1Q66U]akUk ' ? A AA9	  	  	  
7 
7G x~~g666GG
77 7 7 7 7 7 7 7 7 7 7 7 7 7 7 s   BE++E/2E/)NN)r3   r4   r5   r   r   r   r   r   r8   r   rP   r_   rL   r   r'   r&   r;   r;   D   s        
 %)"&+ ++ 8X-.+ SM	+
 3-+ + + +$
s 
 
 
 
- - - x  C            r'   r;   )rm   rJ   typingr   r   r    r   r   r	   r
   
formattingr   packaged_modules.csv.csvr   utilsr   utils.typingr   r   abcr   r   r;   r   r'   r&   <module>ry      s$       				 , , , , , , , , , , 4 4 4 4 4 4 4 4 4 4 4 4 $ $ $ $ $ $ * * * * * *       < < < < < < < < & & & & & &4 4 4 4 4, 4 4 4nL L L L L L L L L Lr'   