
    +gd^                     j    d dl mZ d dlZddlmZmZ ddlmZ ddlm	Z	 ddl
mZ  G d	 d
e          ZdS )    )OptionalN   )Features
NamedSplit)DownloadMode)Spark   )AbstractDatasetReaderc                   z     e Zd ZdZ	 	 	 	 	 ddej        j        dee         dee	         de
d	ed
e
f fdZd Z xZS )SparkDatasetReaderzA dataset reader that reads from a Spark DataFrame.

    When caching, cache materialization is parallelized over Spark; an NFS that is accessible to the driver must be
    provided. Streaming is not currently supported.
    NTarrowdfsplitfeatures	cache_dirload_from_cache_filefile_formatc                      t                      j        d|||d| || _        || _        t	          d|||d|| _        d S )N)r   r   r   )r   r   r    )super__init___load_from_cache_file_file_formatr   builder)	selfr   r   r   r   r   r   kwargs	__class__s	           1lib/python3.11/site-packages/datasets/io/spark.pyr   zSparkDatasetReader.__init__   s     	 	
	
 	
 		
 	
 	
 &:"' 

 
 	
 
    c                     | j         rt          d          | j        rd nt          j        }| j                            || j                   | j                            | j	                  S )Nz%SparkDatasetReader is not streamable.)download_moder   )r   )
	streaming
ValueErrorr   r   FORCE_REDOWNLOADr   download_and_preparer   
as_datasetr   )r   r!   s     r   readzSparkDatasetReader.read+   su    > 	FDEEE $ :]@]))') 	* 	
 	
 	
 |&&TZ&888r   )NNNTr   )__name__
__module____qualname____doc__pysparksql	DataFramer   r   r   strboolr   r'   __classcell__)r   s   @r   r   r      s          '+'+%)"
 
K!
 
#
 8$	

 
 #
 
 
 
 
 
 
2	9 	9 	9 	9 	9 	9 	9r   r   )typingr   r,    r   r   downloadr   packaged_modules.spark.sparkr   abcr
   r   r   r   r   <module>r7      s           # # # # # # # # # # # # # # 0 0 0 0 0 0 & & & & & &)9 )9 )9 )9 )9. )9 )9 )9 )9 )9r   