
    +gd=                         d dl Z d dlmZmZmZ ddlmZmZmZ ddl	m
Z
 ddlmZ ddlmZ dd	lmZ erd dlZd dlZ G d
 de          Z G d d          ZdS )    N)TYPE_CHECKINGOptionalUnion   )DatasetFeaturesconfig)query_table)Sql)logging   )AbstractDatasetInputStreamc                   r     e Zd Z	 	 	 ddeedf         deedddf         d	ee         d
edef
 fdZd Z	 xZ
S )SqlDatasetReaderNFsqlzsqlalchemy.sql.Selectableconsqlalchemy.engine.Connectionsqlalchemy.engine.Enginesqlite3.Connectionfeatures	cache_dirkeep_in_memoryc                 n     t                      j        d|||d| t          d||||d|| _        d S )N)r   r   r   )r   r   r   r    )super__init__r   builder)selfr   r   r   r   r   kwargs	__class__s          /lib/python3.11/site-packages/datasets/io/sql.pyr   zSqlDatasetReader.__init__   sb     	i(iP^iibhiii 
	
 

 
 
    c                     d }d }d }d }| j                             ||||           | j                             d|| j                  }|S )N)download_configdownload_modeverification_mode	base_pathtrain)splitr&   	in_memory)r   download_and_prepare
as_datasetr   )r   r$   r%   r&   r'   datasets         r!   readzSqlDatasetReader.read$   sr     	))+'/ 	* 	
 	
 	
 ,))->$J] * 
 
 r"   )NNF)__name__
__module____qualname__r   strr   r   boolr   r.   __classcell__)r    s   @r!   r   r      s        
 (,$
 
3334
 368RThhi
 8$	

 
 
 
 
 
 
 
$      r"   r   c                   z    e Zd Z	 	 ddededeedddf         dee         d	ee         f
d
ZdefdZ	d Z
defdZdS )SqlDatasetWriterNr-   namer   r   r   r   
batch_sizenum_procc                     ||dk    rt          d| d          || _        || _        || _        |r|nt          j        | _        || _        || _        d S )Nr   z	num_proc z must be an integer > 0.)	
ValueErrorr-   r7   r   r	   DEFAULT_MAX_BATCH_SIZEr8   r9   to_sql_kwargs)r   r-   r7   r   r8   r9   r=   s          r!   r   zSqlDatasetWriter.__init__:   si     HMMKKKKLLL	(2U**8U *r"   returnc                     | j                             dd           }| j                             dd           }| j                             dd          } | j        dd|i| j         }|S )Nr   r   indexFr   )r=   pop_write)r   _r@   writtens       r!   writezSqlDatasetWriter.writeM   sn    ""5$//""5$//"&&w66$+@@E@T-?@@r"   c                 &   |\  }}}|dk    ri |ddin|}t          | j        j        t          ||| j        z             | j        j                  }|                                } |j        | j        | j	        fd|i|}|pt          |          S )Nr   	if_existsappend)tablekeyindicesr@   )r
   r-   dataslicer8   _indices	to_pandasto_sqlr7   r   len)r   argsoffsetr@   r=   batchdfnum_rowss           r!   
_batch_sqlzSqlDatasetWriter._batch_sqlU   s    '+$}DJQJJ@=@+x@@@Ta,#fft677L)
 
 

 __29TYOOOOO"3r77"r"   c                    d}| j         | j         dk    rot          j        t          dt	          | j                  | j                  dt          j                     d          D ]}||                     |f          z  }nt	          | j                  | j        }}t          j
        | j                   5 }t          j        |                    | j        fdt          d||          D                       ||z  r||z  dz   n||z  dt          j                     d          D ]}||z  }	 ddd           n# 1 swxY w Y   |S )	zWrites the pyarrow table as SQL to a database.

        Caller is responsible for opening and closing the SQL connection.
        r   Nr   bazCreating SQL from Arrow format)unitdisabledescc                     g | ]}|fS r   r   ).0rS   r@   r=   s     r!   
<listcomp>z+SqlDatasetWriter._write.<locals>.<listcomp>v   s     eeeF&%7eeer"   )totalrZ   r[   r\   )r9   r   tqdmrangerQ   r-   r8   is_progress_bar_enabledrW   multiprocessingPoolimap)r   r@   r=   rD   rS   rV   r8   pools    ``     r!   rB   zSqlDatasetWriter._writea   s   
 = DMQ$6$6!,aT\**DO<<#;===5	   K K 4??FE=+IJJJK $'t|#4#4dojH %dm44 ( 'IIeeeeeeAxYcFdFdeee  ;CZ:Ok8z1Q66U]akUk ' ? A AA9	! 	! 	! 
( 
(H x'GG
(( ( ( ( ( ( ( ( ( ( ( ( ( ( ( s   :A9E  EE)NN)r/   r0   r1   r   r2   r   r   intr   rE   rW   rB   r   r"   r!   r6   r6   9   s         %)"&+ ++ + 368RThhi	+
 SM+ 3-+ + + +&s    
# 
# 
#      r"   r6   )rd   typingr   r   r    r   r   r	   
formattingr
   packaged_modules.sql.sqlr   utilsr   abcr   sqlite3
sqlalchemyr   r6   r   r"   r!   <module>rq      s       1 1 1 1 1 1 1 1 1 1 ( ( ( ( ( ( ( ( ( ( $ $ $ $ $ $ * * * * * *       + + + + + +  NNN% % % % %1 % % %PF F F F F F F F F Fr"   