
    +gd                     x    d dl Z d dlmZ d dlmZ ddlmZ dZ ee          Z	 G d de          Z
dd
ZddZdS )    N)FileSystems)Pipeline   )
get_loggeri    c                       e Zd ZdZd ZdS )BeamPipelinez<Wrapper over `apache_beam.pipeline.Pipeline` for conveniencec                 b    | j                                                             d          }|dv S )Nrunner)NDirectRunnerPortableRunner)_optionsget_all_optionsget)selfr
   s     9lib/python3.11/site-packages/datasets/utils/beam_utils.pyis_localzBeamPipeline.is_local   s/    ..0044X>>AAA    N)__name__
__module____qualname____doc__r    r   r   r   r      s.        FFB B B B Br   r   Fc                    t           }|                    |          rA|rt                              d| d           n t                              d| d           dS |                    |          5 }t          | d          5 }|                    t                    }|r1|                    |           |                    t                    }|1ddd           n# 1 swxY w Y   ddd           dS # 1 swxY w Y   dS )zJUse the Beam Filesystems to upload to a remote directory on gcs/s3/hdfs...zRemote path already exist: &. Overwriting it as force_upload=True.$. Skipping it as force_upload=False.Nrb)	r   existsloggerinfocreateopenread
CHUNK_SIZEwrite)local_file_pathremote_file_pathforce_uploadfsremote_file
local_filechunks          r   upload_local_to_remoter,      s   	B	yy!""  	KKn6FnnnooooKKl6FlllmmmF	#	$	$ 4/4(( 	4JOOJ//E 4!!%((("
33  4	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	44 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4s7   3C6ACC6C"	"C6%C"	&C66C:=C:c                    t           }t          j                            |          rA|rt                              d|  d           n t                              d|  d           dS |                    |           5 }t          |d          5 }|                    t                    }|r1|	                    |           |                    t                    }|1ddd           n# 1 swxY w Y   ddd           dS # 1 swxY w Y   dS )zNUse the Beam Filesystems to download from a remote directory on gcs/s3/hdfs...zLocal path already exist: r   r   Nwb)
r   ospathr   r   r   r!   r"   r#   r$   )r&   r%   force_downloadr(   r)   r*   r+   s          r   download_remote_to_localr2   &   s   	B	w~~o&&  	KKm5EmmmnnnnKKk5EkkklllF	!	"	" 5k/4(( 	5J$$Z00E 5  '''#((44  5	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	55 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5s7   =D AC(D (C,	,D /C,	0D  DD)F)r/   apache_beam.io.filesystemsr   apache_beam.pipeliner   loggingr   r#   r   r   r   r,   r2   r   r   r   <module>r6      s    				 2 2 2 2 2 2 ) ) ) ) ) )       
	H		B B B B B8 B B B4 4 4 4"5 5 5 5 5 5r   