+gdsdddlZddlZddlZddlZddlmZddlmZddlm Z m Z m Z m Z m Z mZmZmZddlZddlZddlmZddlmZddlmZe rddlmZmZeeZ d Z!d e"d ej#fd Z$d ej%d ej#fdZ&d e"d ej#fdZ'dej#d e"d e(fdZ)de*fdZ+de e(de(d e(fdZ,GddZ-Gdde-Z#Gdde#Z.Gdde.Z/ee"e0e*fZ1Gdd e.Z2ed!e.e e.e e e.Z3Gd"d#e#Z4dAd$e e#d%e(d e#fd&Z5de#d e e"fd'Z6d(Z7d)ej8d e9fd*Z:d+e ej;fd,Ze7d.ej;d0d1fd3Z?dej#d4d5fd6Z@dej#d7ejAfd8ZBdej#fd9ZCdej#d7ejAfd:ZDdej#fd;ZEdej#d.wrappers^'/AW2::]_` Z''AA %'8 9 9 D!3!CB  )rrs` r inject_arrow_table_documentationr!s# Nrfilenamereturnctj|}tj|}|}|SN)pa input_streamipc open_streamread_all)r"in_memory_stream opened_streampa_tables r _in_memory_arrow_table_from_filer.#s>x00F&&'788M%%''H Orbufferctj|}tj|}|}|Sr%)r& BufferReaderr(r)r*)r/streamr,tables r"_in_memory_arrow_table_from_bufferr4*s< _V $ $FF&&v..M  " " $ $E Lrctj|}tj|}|}|Sr%)r& memory_mapr(r)r*)r"memory_mapped_streamr,r-s r$_memory_mapped_arrow_table_from_filer81s>=22F&&';<z'_write_table_to_file..?s$55E5<555555r)openr&RecordBatchStreamWriterr< to_batches write_batchclosesum)r3r"r;writerbatchesr@s r_write_table_to_filerJ8s h  6+elKKK(-(8(8(:(: & &E   u % % % % 55W55555 666666666666666666sA6BBBmemoc |j}||}||t|<|jD])\}}t ||t j||*|S)z!deepcopy a regular class instance) __class____new__id__dict__itemssetattrcopydeepcopy)xrKclsresultkvs r _deepcopyrZBsp +C [[  FDAK   ""3314=D112222 MrarrrUcdt|dz }}||kr|||cxkr ||krnn|||z |||z z||||z zz}|||cxkr||dzkrnn|S|||kr|dz|}}n||}}||kr|||cxkr ||k~ntd|dt|r|dndd)aa Return the position i of a sorted array so that arr[i] <= x < arr[i+1] Args: arr (`List[int]`): non-empty sorted list of integers x (`int`): query Returns: `int`: the position i so that arr[i] <= x < arr[i+1] Raises: `IndexError`: if the array is empty or if the query is outside the array values rr zInvalid query 'z ' for size none.)len IndexError)r[rUijrXs r_interpolation_searchrdLs5 c#hhlqA a%%CFa((((#a&((((( !a%AAJ'CFSVO< = q6Q # # # #QU # # # # #H VaZZq5!qAAaqA a%%CFa((((#a&(((( WqWWC5TSWWfWWW X XXrc|eZdZdejfdZdeeee j fdejfdZ d dejfdZ dS) IndexedTableMixinr3c|j|_d|D|_t jdgd|jDztj|_dS)Nc8g|]}t|dk|Srr`)r? recordbatchs r z.IndexedTableMixin.__init__..is2/ / / '[AQAQTUAUAUKAUAUAUrrc,g|]}t|Sr rj)r?bs rrlz.IndexedTableMixin.__init__..ls4S4S4SSVV4S4S4Sr)dtype)r<_schemarD_batchesnpcumsumint64_offsets)selfr3s r__init__zIndexedTableMixin.__init__gso"', / / +0+;+;+=+=/ / /  %'Iqc4S4ST]4S4S4S.S[][c$d$d$d rindicesr#ct|stdtjj|ddz }t jfdt||Dj S)a' Create a pa.Table by gathering the records at the records at the specified indices. Should be faster than pa.concat_tables(table.fast_slice(int(i) % table.num_rows, 1) for i in indices) since NumPy can compute the binary searches in parallel, highly optimized C zIndices must be non-emptyright)sider ctg|]4\}}j||j|z d5S)r )rqsliceru)r? batch_idxrbrvs rrlz1IndexedTableMixin.fast_gather..xsP    Iq i(..q4=3K/KQOO   rr<) r` ValueErrorrr searchsortedrur&r from_batchesziprp)rvrx batch_indicess` r fast_gatherzIndexedTableMixin.fast_gatherns 7|| :899 9 wWMMMPQQ x$$    $' w$?$?   < %   rrNc|dkrtd||jdks|,|dkr&tjg|jSt |j|}|||z|jdkr<|j|d}|d||j|z |d<nt |j||zdz }|j||dz}|dd||z|j|z |d<|d||j|z |d<tj||jS)aJ Slice the Table using interpolation search. The behavior is the same as `pyarrow.Table.slice` but it's significantly faster. Interpolation search is used to find the start and end indexes of the batches we want to keep. The batches to keep are then concatenated to form the sliced Table. rzOffset must be non-negativer]Nrr ) rarur&rrrprdrqr})rvoffsetlengthrbrIrcs r fast_slicezIndexedTableMixin.fast_slices[ A:::;; ; t}R( ( (V-?FaKK8((DL(AA A !$- 8 8 >Vf_ b0AAAmABB'G ))&4=3C*CDDGAJJ%dmVf_q5HIIAmAAI.G!"+++AvqAQ/QRRGBK ))&4=3C*CDDGAJx$$WT\$BBBrrN) __name__ __module__ __qualname__r&rrwr rintrrndarrayrrr rrrfrffsebheeee 5cBJ)>#? BH    "CC28CCCCCCrrfceZdZdZdejffd ZdefdZdZ dZ dZ d Z d Z d Zd Zd ZdZd.deefdZdZdZdZedZedZedZedZedZedZedZdZ dZ!dZ"dZ#d Z$d!Z%d"Z&d#Z'd$Z(d%Z)d&Z*d'Z+d(Z,d)Z-d*Z.d+Z/d,Z0d-Z1xZ2S)/ra Wraps a pyarrow Table by using composition. This is the base class for `InMemoryTable`, `MemoryMappedTable` and `ConcatenationTable`. It implements all the basic attributes/methods of the pyarrow Table class except the Table transforms: `slice, filter, flatten, combine_chunks, cast, add_column, append_column, remove_column, set_column, rename_columns` and `drop`. The implementation of these methods differs for the subclasses. r3cXt|||_dSr%)superrwr3)rvr3rMs rrwzTable.__init__s&  rrKc|j|t|j<t|j|t|j<t ||Sr%)r3rOlistrqrZ)rvrKs r __deepcopy__zTable.__deepcopy__sF $zR ^^"&t}"5"5R  t$$$rc*|jjtjkrq|j}t jddd5}|j}td|t||d|icdddS#1swxYwYdSd|jiS) Nr:Fz.arrow)deletesuffixzQAttempting to pickle a table bigger than 4GiB. Writing it on the disk instead at )r3r"pathr3) r3r>rMAX_TABLE_NBYTES_FOR_PICKLINGtempfileNamedTemporaryFilenameloggerdebugrJ)rvr3tmp_filer"s r __getstate__zTable.__getstate__s :  D D DJE,T%QQQ *U]#= rhprr%58DDDD)  * * * * * * * * * * * * * * * * * *TZ( (s9A??BBc d|vrf|d}td|t|}td|tj|n|d}t ||dS)Nrz(Unpickling a big table from the disk at z!Removing temporary table file at r3)rrr.osremoverrw)rvstater"r3s r __setstate__zTable.__setstate__s U??V}H LLNHNN O O O4X>>E LLGXGG H H H Ih    'NE tU#####rc&|jj|i|S)a Perform validation checks. An exception is raised if validation fails. By default only cheap validation checks are run. Pass `full=True` for thorough validation checks (potentially `O(n)`). Args: full (`bool`, defaults to `False`): If `True`, run expensive checks, otherwise cheap checks only. Raises: `pa.lib.ArrowInvalid`: if validation fails )r3validatervargskwargss rrzTable.validates#tz"D3F333rcptd|D}d|D}|jj|i|S)aN Check if contents of two tables are equal. Args: other ([`~datasets.table.Table`]): Table to compare against. check_metadata `bool`, defaults to `False`): Whether schema metadata equality should be checked as well. Returns: `bool` c3RK|]"}t|tr|jn|V#dSr% isinstancerr3)r?args rrAzTable.equals..s7RRc*S%"8"8ASYYcRRRRRRrcRi|]$\}}|t|tr|jn|%Sr r)r?rXrYs r z Table.equals..s3OOO1! 1e 4 4;QWW!OOOr)tupler3equalsrs rrz Table.equalssMRRTRRRRROOOOO tz $1&111rc&|jj|i|S)ak Convert Table to list of (contiguous) `RecordBatch` objects. Args: max_chunksize (`int`, defaults to `None`): Maximum size for `RecordBatch` chunks. Individual chunks may be smaller depending on the chunk layout of individual columns. Returns: `List[pyarrow.RecordBatch]` )r3rDrs rrDzTable.to_batchess%tz$d5f555rc&|jj|i|S)zf Convert the Table to a `dict` or `OrderedDict`. Returns: `dict` )r3 to_pydictrs rrzTable.to_pydicts$tz#T4V444rc |jj|i|S#t$rB|jj|i|fdt t |jDcYSwxYw)zR Convert the Table to a list Returns: `list` c0g|]fdDS)c.i|]}||Sr r )r?rXrbpydicts rrz.Table.to_pylist... s#555Qq ! 555rr )r?rbrs @rrlz#Table.to_pylist.. s2VVV!55555f555VVVr)r3 to_pylistAttributeErrorrranger`)rvrrrs @rrzTable.to_pylists W'4:'888 8 W W W)TZ)4:6::FVVVVuS__?U?UVVV V V V WsA A! A!c&|jj|i|S)a Convert to a pandas-compatible NumPy array or DataFrame, as appropriate. Args: memory_pool (`MemoryPool`, defaults to `None`): Arrow MemoryPool to use for allocations. Uses the default memory pool is not passed. strings_to_categorical (`bool`, defaults to `False`): Encode string (UTF8) and binary types to `pandas.Categorical`. categories (`list`, defaults to `empty`): List of fields that should be returned as `pandas.Categorical`. Only applies to table-like data structures. zero_copy_only (`bool`, defaults to `False`): Raise an `ArrowException` if this function call would require copying the underlying data. integer_object_nulls (`bool`, defaults to `False`): Cast integers with nulls to objects. date_as_object (`bool`, defaults to `True`): Cast dates to objects. If `False`, convert to `datetime64[ns]` dtype. timestamp_as_object (`bool`, defaults to `False`): Cast non-nanosecond timestamps (`np.datetime64`) to objects. This is useful if you have timestamps that don't fit in the normal date range of nanosecond timestamps (1678 CE-2262 CE). If `False`, all timestamps are converted to `datetime64[ns]` dtype. use_threads (`bool`, defaults to `True`): Whether to parallelize the conversion using multiple threads. deduplicate_objects (`bool`, defaults to `False`): Do not create multiple copies Python objects when created, to save on memory use. Conversion will be slower. ignore_metadata (`bool`, defaults to `False`): If `True`, do not use the 'pandas' metadata to reconstruct the DataFrame index, if present. safe (`bool`, defaults to `True`): For certain data types, a cast is needed in order to store the data in a pandas DataFrame or Series (e.g. timestamps are always stored as nanoseconds in pandas). This option controls whether it is a safe cast or not. split_blocks (`bool`, defaults to `False`): If `True`, generate one internal "block" for each column when creating a pandas.DataFrame from a `RecordBatch` or `Table`. While this can temporarily reduce memory note that various pandas operations can trigger "consolidation" which may balloon memory use. self_destruct (`bool`, defaults to `False`): EXPERIMENTAL: If `True`, attempt to deallocate the originating Arrow memory while converting the Arrow object to pandas. If you use the object after calling `to_pandas` with this option it will crash your program. types_mapper (`function`, defaults to `None`): A function mapping a pyarrow DataType to a pandas `ExtensionDtype`. This can be used to override the default pandas type for conversion of built-in pyarrow types or in absence of `pandas_metadata` in the Table schema. The function receives a pyarrow DataType and is expected to return a pandas `ExtensionDtype` or `None` if the default conversion should be used for that type. If you have a dictionary mapping, you can pass `dict.get` as function. Returns: `pandas.Series` or `pandas.DataFrame`: `pandas.Series` or `pandas.DataFrame` depending on type of object )r3 to_pandasrs rrzTable.to_pandassx$tz#T4V444rc&|jj|i|Sr%)r3 to_stringrs rrzTable.to_stringMs#tz#T4V444rN max_chunksizec8|j|S)a Convert the Table to a RecordBatchReader. Note that this method is zero-copy, it merely exposes the same data under a different API. Args: max_chunksize (`int`, defaults to `None`) Maximum size for RecordBatch chunks. Individual chunks may be smaller depending on the chunk layout of individual columns. Returns: `pyarrow.RecordBatchReader` r)r3 to_reader)rvrs rrzTable.to_readerPsz##-#@@@rc&|jj|i|S)z Select a schema field by its column name or numeric index. Args: i (`Union[int, str]`): The index or name of the field to retrieve. Returns: `pyarrow.Field` )r3fieldrs rrz Table.field`s tz0000rc&|jj|i|S)z Select a column by its column name, or numeric index. Args: i (`Union[int, str]`): The index or name of the column to retrieve. Returns: `pyarrow.ChunkedArray` )r3columnrs rrz Table.columnms!tz $1&111rc&|jj|i|S)zy Iterator over all columns in their numerical order. Yields: `pyarrow.ChunkedArray` )r3 itercolumnsrs rrzTable.itercolumnszs&tz%t6v666rc|jjS)ze Schema of the table and its columns. Returns: `pyarrow.Schema` r3r<rvs rr<z Table.schemasz  rc|jjS)zo List of all columns in numerical order. Returns: `List[pa.ChunkedArray]` )r3columnsrs rrz Table.columnssz!!rc|jjS)zT Number of columns in this table. Returns: int )r3 num_columnsrs rrzTable.num_columnssz%%rc|jjS)z Number of rows in this table. Due to the definition of a table, all columns have the same number of rows. Returns: int )r3num_rowsrs rrzTable.num_rowssz""rc|jjS)z Dimensions of the table: (#rows, #columns). Returns: `(int, int)`: Number of rows and number of columns. )r3shapers rrz Table.shapeszrc|jjS)zN Total number of bytes consumed by the elements of the table. )r3r>rs rr>z Table.nbytess z  rc|jjS)z/ Names of the table's columns. )r3 column_namesrs rrzTable.column_namess z&&rc,||Sr%)r)rvothers r__eq__z Table.__eq__s{{5!!!rc|j|Sr%r3)rvrbs r __getitem__zTable.__getitem__sz!}rc*t|jSr%)r`r3rs r__len__z Table.__len__s4:rcp|jd|jjSNr)r3__repr__rrMrrs rrzTable.__repr__s+z""$$,,_dn>UVVVrcp|jd|jjSr)r3__str__rrMrrs rrz Table.__str__s+z!!##++OT^=TUUUrct)t Compute zero-copy slice of this Table. Args: offset (`int`, defaults to `0`): Offset from start of table to slice. length (`int`, defaults to `None`): Length of slice (default is until end of table starting from offset). Returns: `datasets.table.Table` NotImplementedErrorrs rr}z Table.slice"###rct[ Select records from a Table. See `pyarrow.compute.filter` for full usage. rrs rfilterz Table.filter"###rctp Flatten this Table. Each column with a struct type is flattened into one column per struct field. Other columns are left unchanged. Args: memory_pool (`MemoryPool`, defaults to `None`): For memory allocations, if required, otherwise use default pool. Returns: `datasets.table.Table` rrs rflattenz Table.flatten"###rct Make a new table by combining the chunks this table has. All the underlying chunks in the `ChunkedArray` of each column are concatenated into zero or one chunk. Args: memory_pool (`MemoryPool`, defaults to `None`): For memory allocations, if required, otherwise use default pool. Returns: `datasets.table.Table` rrs rcombine_chunkszTable.combine_chunksrrct] Cast table values to another schema. Args: target_schema (`Schema`): Schema to cast to, the names and order of fields must match. safe (`bool`, defaults to `True`): Check for overflows or other unsafe conversions. Returns: `datasets.table.Table` rrs rcastz Table.cast "###rct)aP EXPERIMENTAL: Create shallow copy of table by replacing schema key-value metadata with the indicated new metadata (which may be None, which deletes any existing metadata Args: metadata (`dict`, defaults to `None`): Returns: `datasets.table.Table`: shallow_copy rrs rreplace_schema_metadatazTable.replace_schema_metadatarrctG Add column to Table at position. A new table is returned with the column added, the original table object is left unchanged. Args: i (`int`): Index to place the column at. field_ (`Union[str, pyarrow.Field]`): If a string is passed then the type is deduced from the column data. column (`Union[pyarrow.Array, List[pyarrow.Array]]`): Column data. Returns: `datasets.table.Table`: New table with the passed column added. rrs r add_columnzTable.add_column&&"###rct)a Append column at end of columns. Args: field_ (`Union[str, pyarrow.Field]`): If a string is passed then the type is deduced from the column data. column (`Union[pyarrow.Array, List[pyarrow.Array]]`): Column data. Returns: `datasets.table.Table`: New table with the passed column added. rrs r append_columnzTable.append_column;rrct)z Create new Table with the indicated column removed. Args: i (`int`): Index of column to remove. Returns: `datasets.table.Table`: New table without the column. rrs r remove_columnzTable.remove_columnKs"###rct)a Replace column in Table at position. Args: i (`int`): Index to place the column at. field_ (`Union[str, pyarrow.Field]`): If a string is passed then the type is deduced from the column data. column (`Union[pyarrow.Array, List[pyarrow.Array]]`): Column data. Returns: `datasets.table.Table`: New table with the passed column set. rrs r set_columnzTable.set_columnXs "###rctJ Create new table with columns renamed to provided names. rrs rrename_columnszTable.rename_columnsjrrct)ai Drop one or more columns and return a new table. Args: columns (`List[str]`): List of field names referencing existing columns. Raises: `KeyError` : if any of the passed columns name are not existing. Returns: `datasets.table.Table`: New table without the columns. rrs rdropz Table.dropprrct)ae Select columns of the table. Returns a new table with the specified columns, and metadata preserved. Args: columns (:obj:`Union[List[str], List[int]]`): The column names or integer indices to select. Returns: `datasets.table.Table`: table with only a subset of the columns rrs rselectz Table.selectrrr%)3rrrrr&rrwdictrrrrrrDrrrrr rrrrrpropertyr<rrrrr>rrrrrrr}rrrrrrrr r rrr __classcell__rMs@rrrs  bh%%%%% ) ) ) $ $ $444 222" 6 6 6555 W W W<5<5<5|555AAx}AAAA 1 1 1 2 2 2777!!X!""X"&&X& # #X #  X !!X! ''X' """WWWVVV$$$ $$$ $ $ $$$$ $ $ $ $ $ $$$$*$$$ $ $ $$$$$$$$ $$$ $ $ $ $ $ $ $rrceZdZdZdS) TableBlockz `TableBlock` is the allowed class inside a `ConcanetationTable`. Only `MemoryMappedTable` and `InMemoryTable` are `TableBlock`. This is because we don't want a `ConcanetationTable` made out of other `ConcanetationTables`. N)rrrrr rrrrs  DrrceZdZdZedefdZedejfdZ edZ edZ edZ ed Z ed Zdd ZdZdZdZdZdZdZdZdZdZdZdZdZd S) InMemoryTablea_ The table is said in-memory when it is loaded into the user's RAM. Pickling it does copy all the data using memory. Its implementation is simple and uses the underlying pyarrow Table methods directly. This is different from the `MemoryMapped` table, for which pickling doesn't copy all the data in memory. For a `MemoryMapped`, unpickling instead reloads the table from the disk. `InMemoryTable` must be used when data fit in memory, while `MemoryMapped` are reserved for data bigger than memory or when you want the memory footprint of your application to stay low. r"c6t|}||Sr%)r.)rVr"r3s r from_filezInMemoryTable.from_files0::s5zzrr/c6t|}||Sr%)r4)rVr/r3s r from_bufferzInMemoryTable.from_buffers26::s5zzrc@|tjj|i|S)aG Convert pandas.DataFrame to an Arrow Table. The column types in the resulting Arrow Table are inferred from the dtypes of the pandas.Series in the DataFrame. In the case of non-object Series, the NumPy dtype is translated to its Arrow equivalent. In the case of `object`, we need to guess the datatype by looking at the Python objects in this Series. Be aware that Series of the `object` dtype don't carry enough information to always lead to a meaningful Arrow type. In the case that we cannot infer a type, e.g. because the DataFrame is of length 0 or the Series only contains `None/nan` objects, the type is set to null. This behavior can be avoided by constructing an explicit schema and passing it to this function. Args: df (`pandas.DataFrame`): schema (`pyarrow.Schema`, *optional*): The expected schema of the Arrow Table. This can be used to indicate the type of columns if we cannot infer it automatically. If passed, the output will have exactly this schema. Columns specified in the schema that are not found in the DataFrame columns or its index will raise an error. Additional columns or index levels in the DataFrame which are not specified in the schema will be ignored. preserve_index (`bool`, *optional*): Whether to store the index as an additional column in the resulting `Table`. The default of None will store the index as a column, except for RangeIndex which is stored as metadata only. Use `preserve_index=True` to force it to be stored as a column. nthreads (`int`, defaults to `None` (may use up to system CPU count threads)) If greater than 1, convert columns to Arrow in parallel using indicated number of threads. columns (`List[str]`, *optional*): List of column to be converted. If `None`, use all columns. safe (`bool`, defaults to `True`): Check for overflows or other unsafe conversions, Returns: `datasets.table.Table`: Examples: ```python >>> import pandas as pd >>> import pyarrow as pa >>> df = pd.DataFrame({ ... 'int': [1, 2], ... 'str': ['a', 'b'] ... }) >>> pa.Table.from_pandas(df) ``` )r&r from_pandasrVrrs rr"zInMemoryTable.from_pandass'ps28'888999rc@|tjj|i|S)a Construct a Table from Arrow arrays. Args: arrays (`List[Union[pyarrow.Array, pyarrow.ChunkedArray]]`): Equal-length arrays that should form the table. names (`List[str]`, *optional*): Names for the table columns. If not passed, schema must be passed. schema (`Schema`, defaults to `None`): Schema for the created table. If not passed, names must be passed. metadata (`Union[dict, Mapping]`, defaults to `None`): Optional metadata for the schema (if inferred). Returns: `datasets.table.Table` )r&r from_arraysr#s rr%zInMemoryTable.from_arrayss&$s28'888999rc@|tjj|i|S)a Construct a Table from Arrow arrays or columns. Args: mapping (`Union[dict, Mapping]`): A mapping of strings to Arrays or Python lists. schema (`Schema`, defaults to `None`): If not passed, will be inferred from the Mapping values metadata (`Union[dict, Mapping]`, defaults to `None`): Optional metadata for the schema (if inferred). Returns: `datasets.table.Table` )r&r from_pydictr#s rr'zInMemoryTable.from_pydicts& s28'888999rcH|tjj|g|Ri|S)a Construct a Table from list of rows / dictionaries. Args: mapping (`List[dict]`): A mapping of strings to row values. schema (`Schema`, defaults to `None`): If not passed, will be inferred from the Mapping values metadata (`Union[dict, Mapping]`, defaults to `None`): Optional metadata for the schema (if inferred). Returns: `datasets.table.Table` )r&r from_pylist)rVmappingrrs rr)zInMemoryTable.from_pylists1 s28'A$AAA&AABBBrc@|tjj|i|S)a Construct a Table from a sequence or iterator of Arrow `RecordBatches`. Args: batches (`Union[Sequence[pyarrow.RecordBatch], Iterator[pyarrow.RecordBatch]]`): Sequence of `RecordBatch` to be converted, all schemas must be equal. schema (`Schema`, defaults to `None`): If not passed, will be inferred from the first `RecordBatch`. Returns: `datasets.table.Table`: )r&rrr#s rrzInMemoryTable.from_batches%s&s28($9&99:::rrNcJt|||S)rrr)rr)rvrrs rr}zInMemoryTable.slice5s"T__F6_JJKKKrc@t|jj|i|Sr)rr3rrs rrzInMemoryTable.filterFs'.TZ.???@@@rcFtt|jg|Ri|Sr)r table_flattenr3rs rrzInMemoryTable.flattenLs,]4:GGGGGGHHHrc@t|jj|i|Sr)rr3rrs rrzInMemoryTable.combine_chunksZs'6TZ6GGGHHHrcFtt|jg|Ri|Sr)r table_castr3rs rrzInMemoryTable.castjs,Z DTDDDVDDEEErc@t|jj|i|S)T EXPERIMENTAL: Create shallow copy of table by replacing schema key-value metadata with the indicated new metadata (which may be `None`, which deletes any existing metadata). Args: metadata (`dict`, defaults to `None`): Returns: `datasets.table.Table`: shallow_copy )rr3rrs rrz%InMemoryTable.replace_schema_metadatays'?TZ?PPPQQQrc@t|jj|i|Sr)rr3rrs rrzInMemoryTable.add_columns'&2TZ2DCFCCDDDrc@t|jj|i|S Append column at end of columns. Args: field_ (`Union[str, pyarrow.Field]`): If a string is passed then the type is deduced from the column data. column (`Union[pyarrow.Array, List[pyarrow.Array]]`): Column data. Returns: `datasets.table.Table`: New table with the passed column added. )rr3rrs rrzInMemoryTable.append_columns'5TZ5tFvFFGGGrc@t|jj|i|S) Create new Table with the indicated column removed. Args: i (`int`): Index of column to remove. Returns: `datasets.table.Table`: New table without the column. )rr3r rs rr zInMemoryTable.remove_columns'5TZ5tFvFFGGGrc@t|jj|i|S Replace column in Table at position. Args: i (`int`): Index to place the column at. field_ (`Union[str, pyarrow.Field]`): If a string is passed then the type is deduced from the column data. column (`Union[pyarrow.Array, List[pyarrow.Array]]`): Column data. Returns: `datasets.table.Table`: New table with the passed column set. )rr3r rs rr zInMemoryTable.set_columns'"2TZ2DCFCCDDDrc@t|jj|i|Sr)rr3rrs rrzInMemoryTable.rename_columnss'6TZ6GGGHHHrc@t|jj|i|S)y Drop one or more columns and return a new table. Args: columns (`List[str]`): List of field names referencing existing columns. Raises: `KeyError` : if any of the passed columns name are not existing. Returns: `datasets.table.Table`: New table without the columns. )rr3rrs rrzInMemoryTable.drops%_TZ_d=f==>>>rc@t|jj|i|S) Select columns of the table. Returns a new table with the specified columns, and metadata preserved. Args: columns (:obj:`Union[List[str], List[int]]`): The column names or integer indices to select. Returns: :class:`datasets.table.Table`: New table with the specified columns, and metadata preserved. )rr3rrs rrzInMemoryTable.selects'.TZ.???@@@rr)rrrr classmethodstrrr&Bufferr r"r%r'r)rr}rrrrrrrr r rrrr rrrrs  [[7:7:[7:r::[:&::[:"CC[C" ; ;[ ;LLLL"AAA I I IIII F F F R R REEE*HHH" H H HEEE&III ???" A A A A Arrc <eZdZdZddejdedeee ffd Z e ddefdZ d Z d Zeddejdeee d ejfd Zd e d ee fdZddZdZdZdZdZdZdZdZdZdZdZdZdZxZS)MemoryMappedTableaT The table is said memory mapped when it doesn't use the user's RAM but loads the data from the disk instead. Pickling it doesn't copy the data into memory. Instead, only the path to the memory mapped arrow file is pickled, as well as the list of transforms to "replay" when reloading the table from the disk. Its implementation requires to store an history of all the transforms that were applied to the underlying pyarrow Table, so that they can be "replayed" when reloading the Table from the disk. This is different from the `InMemoryTable` table, for which pickling does copy all the data in memory. `InMemoryTable` must be used when data fit in memory, while `MemoryMapped` are reserved for data bigger than memory or when you want the memory footprint of your application to stay low. Nr3rreplaysct|tj||_||ng|_dSr%)rrwrrabspathrI)rvr3rrIrMs rrwzMemoryMappedTable.__init__sD GOOD)) 070CWW rr"cft|}|||}||||Sr%)r8_apply_replays)rVr"rIr3s rrzMemoryMappedTable.from_files84X>>""5'22s5(G,,,rc |j|jdS)NrrIrOrs rrzMemoryMappedTable.__getstate__s dl;;;rc|d}|d}t|}|||}t||||dS)NrrIrO)r8rMrHrw)rvrrrIr3s rrzMemoryMappedTable.__setstate__sZV} "4T::##E733""4T7"KKKKKrr#c|M|D]J\}}}|dkrt|g|Ri|}|dkrt|g|Ri|}4t|||i|}K|S)Nrr)r3r0getattr)r3rIrrrs rrMz MemoryMappedTable._apply_replays#s  &- B B"dF6>>&u>t>>>v>>EEY&&)%A$AAA&AAEE0GE400$A&AAEE rreplaycbtj|j}|||Sr%)rSrTrIappend)rvrSrIs r_append_replayz MemoryMappedTable._append_replay/s*- --vrrcd||fif}||}t||||j|S)rr}r-)rVrHrr)rvrrrSrIs rr}zMemoryMappedTable.slice4sOFF+R0%%f-- v!N!NPTPY[bcccrcdtj|tj|f}||}t|jj|i||j|S)rr)rSrTrVrHr3rrrvrrrSrIs rrzMemoryMappedTable.filterGsaDM$//v1F1FG%%f-- !2!2D!CF!C!CTYPWXXXrcdtj|tj|f}||}tt |jg|Ri||j|S)rr)rSrTrVrHr0r3rrYs rrzMemoryMappedTable.flattenOsfT]400$-2G2GH%%f-- tz!KD!K!K!KF!K!KTYX_```rcdtj|tj|f}||}t|jj|i||j|S)a Make a new table by combining the chunks this table has. All the underlying chunks in the ChunkedArray of each column are concatenated into zero or one chunk. Args: memory_pool (`MemoryPool`, defaults to `None`): For memory allocations, if required, otherwise use default pool. Returns: `datasets.table.Table` r)rSrTrVrHr3rrrYs rrz MemoryMappedTable.combine_chunks_sa#DM$$7$7v9N9NO%%f-- !:!:D!KF!K!KTYX_```rcdtj|tj|f}||}tt |jg|Ri||j|S)a\ Cast table values to another schema Args: target_schema (`Schema`): Schema to cast to, the names and order of fields must match. safe (`bool`, defaults to `True`): Check for overflows or other unsafe conversions. Returns: `datasets.table.Table` r)rSrTrVrHr3r3rrYs rrzMemoryMappedTable.castqsf$---t}V/D/DE%%f-- DJ!H!H!H!H!H!H$)U\]]]rcdtj|tj|f}||}t|jj|i||j|S)aQ EXPERIMENTAL: Create shallow copy of table by replacing schema key-value metadata with the indicated new metadata (which may be None, which deletes any existing metadata. Args: metadata (`dict`, defaults to `None`): Returns: `datasets.table.Table`: shallow_copy r)rSrTrVrHr3rrrYs rrz)MemoryMappedTable.replace_schema_metadatasd,T]4-@-@$-PVBWBWX%%f-- !C!CT!TV!T!TVZV_ahiiircdtj|tj|f}||}t|jj|i||j|S)rr)rSrTrVrHr3rrrYs rrzMemoryMappedTable.add_columnsa& d 3 3T]65J5JK%%f-- !6!6!G!G!GT[\\\rcdtj|tj|f}||}t|jj|i||j|S)r9r)rSrTrVrHr3rrrYs rrzMemoryMappedTable.append_columnsa"4=#6#6 f8M8MN%%f-- !9!94!J6!J!JDIW^___rcdtj|tj|f}||}t|jj|i||j|S)r;r )rSrTrVrHr3r rrYs rr zMemoryMappedTable.remove_columnsa"4=#6#6 f8M8MN%%f-- !9!94!J6!J!JDIW^___rcdtj|tj|f}||}t|jj|i||j|S)r>r )rSrTrVrHr3r rrYs rr zMemoryMappedTable.set_columnsa" d 3 3T]65J5JK%%f-- !6!6!G!G!GT[\\\rcdtj|tj|f}||}t|jj|i||j|S)rr)rSrTrVrHr3rrrYs rrz MemoryMappedTable.rename_columnssa#DM$$7$7v9N9NO%%f-- !:!:D!KF!K!KTYX_```rcdtj|tj|f}||}t|jj|i||j|S)rAr)rSrTrVrHr3rrrYs rrzMemoryMappedTable.drops^$---t}V/D/DE%%f-- $!A&!A!A49gVVVrcdtj|tj|f}||}t|jj|i||j|S)rCr)rSrTrVrHr3rrrYs rrzMemoryMappedTable.selectsaDM$//v1F1FG%%f-- !2!2D!CF!C!CTYPWXXXrr%r) rrrrr&rrEr rReplayrwrDrrr staticmethodrMrVr}rrrrrrrr r rrrrrs@rrHrHs&(LLbhLcLHT&\.Fs-[[[5GE7$;$;FU[[[[[rrT)promoter z'axis' must be either 0 or 1)r& concat_tables enumeraterrrrr)rjrs pa_tablesrbr3r-rcols r_concat_blocksz!ConcatenationTable._concat_blocksDs[[TZ[[[ 199#It<<< < QYY%i00 E E566$HH%();U]%K%KEE c#+#9#9$#D#DEO;<< z2ConcatenationTable._merge_blocks..cs:VWYfKgKgr)keyr}c>g|]}|dS)r r}) _merge_blocks)r? row_blockrVs rrlz4ConcatenationTable._merge_blocks..hs,ZZZiS..yq.AAZZZrc3<K|]}t|dkVdS)r Nrj)r?rs rrAz3ConcatenationTable._merge_blocks..is-FF93y>>Q&FFFFFFrcg|] }|D]}| Sr r )r?rblocks rrlz4ConcatenationTable._merge_blocks..ks%QQQyyQQeUQQQQrr)rrr{rallr)rVrjrs merged_blocks is_in_memory block_groups` rrz ConcatenationTable._merge_blocks_s  M-4VAgAg-h-h-h 3 3) kd#01C1CDDUDU\`1C1a1a#b#b"cKk!2!22  3 [ZZZSYZZZMFF FFFFF  # 1 1QQMQQQXY!2!! rct|tr|St|dtr||dS||S)Nrr})rrr)rVrjs r_consolidate_blocksz&ConcatenationTable._consolidate_blocksos\ fj ) ) -M q : . . -$$V!$44 4$$V,, ,rcV||}t|tr|}||j|ggSt|dtr/||d}d|D}|||S||}|||S)Nrr}cg|]}|gSr r )r?ts rrlz2ConcatenationTable.from_blocks..s***aqc***r)rrrr3r{rr)rVrjr3s r from_blockszConcatenationTable.from_blocksxs((00 fj ) ) &E3u{eWI.. . q : . . &&&vA&66E**6***F3uf%% %BB6JJE3uf%% %rrc  dttjtfdtttfd}dttdt dt ttttffddtttdtttdt ttttttfffd ddtttdtttd t dtttf fd }||d }|d dD]}||}||||}||S)aCreate `ConcatenationTable` from list of tables. Args: tables (list of `Table` or list of `pyarrow.Table`): List of tables. axis (`{0, 1}`, defaults to `0`, meaning over rows): Axis to concatenate over, where `0` means over rows (vertically) and `1` means over columns (horizontally). r3r#ct|tjrt|ggSt|trt j|jS|ggSr%)rr&rrrirSrTrjrs r to_blocksz1ConcatenationTable.from_tables..to_blockssY%** !&u--.//E#566 !}U\222y rrrcHfdD}fdD}||fS)Nc<g|]}|dSrir})r?r3rs rrlzLConcatenationTable.from_tables.._slice_row_block..s'DDDekk!V,,DDDrc hg|].}|tdz /Sri)r}r`)r?r3rrs rrlzLConcatenationTable.from_tables.._slice_row_block..s8```UVS1->->-GHH```rr )rrsliced remainders`` r_slice_row_blockz8ConcatenationTable.from_tables.._slice_row_blocksEDDDD)DDDF`````V_```I9$ $rrWrjct|t|}}gg}}|r|rt|ddt|ddkro||d|dt|dd\}|d<||nt|ddt|ddkro||d|dt|dd\}|d<||nP||d||d|r||s|rt d||fS)ah Make sure each row_block contain the same num_rows to be able to concatenate them on axis=1. To do so, we modify both blocks sets to have the same row_blocks boundaries. For example, if `result` has 2 row_blocks of 3 rows and `blocks` has 3 row_blocks of 2 rows, we modify both to have 4 row_blocks of size 2, 1, 1 and 2: [ x x x | x x x ] + [ y y | y y | y y ] ----------------------------- = [ x x | x | x | x x ] [ y y | y | y | y y ] rzQFailed to concatenate on axis=1 because tables don't have the same number of rows)rr`rUpopr)rWrj new_result new_blocksrrs r_split_both_likez8ConcatenationTable.from_tables.._split_both_likes""&\\4<._extend_blockss}qyy f%%%%!1!1&&!A!A$-f$5$500LAy1I$$Y////Mrr Nr}ri)r r&rrrrr r) rVrrsrrrjr3 table_blocksrrs @@r from_tableszConcatenationTable.from_tabless !U28U?3 !T*=M8N ! ! ! ! %Z(8 %# %%PTU_P`bfgqbrPrJs % % % % # *j)*# *48j9I4J# * 4Z()4Z0@+AA B# *# *# *# *# *# *LYZ  j)* 48j9I4J RU  $z" #      6!9%%ABBZ E EE$9U++L#^FLtDDDFFv&&&rc#bKd}|jD]"}t|d}||fV||z }#dS)Nr)rjr`)rvrrrs r_sliceszConcatenationTable._slicessRk  F^^F6" " " " f FF  rc|j}n |jz g}|jD]}t |d}dkrnj|kr|z +|zkr-|fd|Dz|z dca|fd|Dd\t ||S)rrNrc:g|]}|Sr r)r?rrs rrlz,ConcatenationTable.slice..s#???1qwwv???rc<g|]}|Sr r)r?rrrs rrlz,ConcatenationTable.slice..s'GGG1qwwvv66GGGr)rr)r3r}rrjr`rUri)rvrrr3rjrn_rowss `` rr}zConcatenationTable.slices    77!-4=63Ik & &F^^F{{6!!&6F?** ???????@@@!'&6!91 GGGGGGGGHHH!%!%000rc |jj|gRi}g}t|j|jD]A\\}}}||| | fd|DBt||S)rc2g|]}|jgRiSr )r)r?rrrsubmasks rrlz-ConcatenationTable.filter..s5NNN!818G=d===f==NNNr)r3rrrrjr}rUri) rvmaskrrr3rjrrrrs `` @rrzConcatenationTable.filters" !$888888(+DL$+(F(F P P $ VVfjj00G MMNNNNNNvNNN O O O O!%000rct|jgRi}g}|jD]$}|fd|D%t ||S)rc*g|]}|jiSr )rr?rrrs rrlz.ConcatenationTable.flatten..s*FFF!919d5f55FFFr)r0r3rjrUrirvrrr3rjrs `` rrzConcatenationTable.flatten szdj:4:::6::k H HF MMFFFFFvFFF G G G G!%000rc|jji}g}|jD]$}|fd|D%t ||S)rc*g|]}|jiSr )rrs rrlz5ConcatenationTable.combine_chunks..-s,MMM+1+T.Hs64o4o81e\a\fjn\n\nQ\n\n\n\n4o4orc6i|]}|j|jSr r)r?subfieldtarget_featuress rrz+ConcatenationTable.cast..Is%'p'p'pZb x}7U'p'p'pr)featuresrr3r3from_arrow_schemarjrrrUrnextrx arrow_schemarri)rv target_schemarrrr3rjrm new_tablesfieldsrn subfields subfeatures subschemarrs @@rrzConcatenationTable.cast0sk '&&&&&4:}FtFFFvFF4(4]CC & &IJ-((F% M M $1rrD$$VZZ4o4o4o4oyQWGXGX4o4o4o0o0o%p%pqqqq&h'p'p'p'pfo'p'p'pqq '4 !!-(- "KD"K"K"KF"K"KLLLL MM* % % % %!%000rc|jji}g}|jD]$}|fd|D%t ||jS)r5c*g|]}|jiSr )rrs rrlz>ConcatenationTable.replace_schema_metadata..^s,VVV!414dEfEEVVVr)r3rrjrUrirs `` rrz*ConcatenationTable.replace_schema_metadataOsv3 2DCFCCk X XF MMVVVVVvVVV W W W W!%555rctrrrs rrzConcatenationTable.add_columnarrctr8rrs rrz ConcatenationTable.append_columnvs"###rc|jj|gRi}|jj|g}|jD]%}|fd|D&t ||S)r;cxg|]6}|jvr)|j|jgRin|7Sr )rr index)r?rrrrs rrlz4ConcatenationTable.remove_column..skUY\]\jTjTjOAOAN$8$8$>$>PPPPPPPpqr)r3r rrjrUri)rvrbrrr3rjrrs `` @rr z ConcatenationTable.remove_columns) (..s"J"J"J45;"J"J"Jr)rr)r?rrrrs rrlz5ConcatenationTable.rename_columns..sPmmmab!!"J"J"J"J1>"J"J"J\T\\\U[\\mmmr)r3rrrrrjrUri)rvrrrr3rjrs ``` rrz!ConcatenationTable.rename_columnss* )%A$AAA&AAS0%8899k  F MMmmmmmmflmmm    "%000rc|jjgRi}g}|jD]%}|fd|D&t ||S)rAcLg|]jfdDgRi S)c&g|] }|jv |Sr rr?crs rrlz6ConcatenationTable.drop...s%"M"M"Man9L9L19L9L9Lr)rr?rrrrs @rrlz+ConcatenationTable.drop..sNpppde616"M"M"M"Mg"M"M"M_PT___X^__pppr)r3rrjrUrirvrrrr3rjrs ``` rrzConcatenationTable.drops  9$999&99k r rF MMppppppioppp q q q q!%000rc|jjgRi}g}|jD]%}|fd|D&t ||S)rCcLg|]jfdDgRi S)c&g|] }|jv |Sr rrs rrlz8ConcatenationTable.select...s%$O$O$O11;N;NQ;N;N;Nr)rrs @rrlz-ConcatenationTable.select..sNrrrfg818$O$O$O$O$O$O$OaRVaaaZ`aarrrr)r3rrjrUrirs ``` rrzConcatenationTable.selects" !';D;;;F;;k t tF MMrrrrrrkqrrr s s s s!%000rrir%r)'rrrrr&rrrrwrrrfr rr{rDrrrgr rrrrrrr}rrrrrrrr r rrrrrs@rriris2 bh T*5E0F      '''@@@ ==tE*bh*>$?@==TVT\===\= JT*EU@VJ[][cJJJ[J  #6 hsm Wj   [ -)<-AT---[- &!4 &9M & & &[ &O'O'eBHeO& 1 1 1111$111(111>666$$$$*$$$"1110$$$& 1 1 1111*1111111rrirrsct|}t|dkr|dSt||S)aH Concatenate tables. Args: tables (list of `Table`): List of tables to be concatenated. axis (`{0, 1}`, defaults to `0`, meaning over rows): Axis to concatenate over, where `0` means over rows (vertically) and `1` means over columns (horizontally). Returns: `datasets.table.Table`: If the number of input tables is > 1, then the returned table is a `datasets.table.ConcatenationTable`. Otherwise if there's only one table, it is returned as is. r rr})rr`rir)rrss rrwrwsC"&\\F 6{{aay  ) )&t ) < <.wrapper..s3$\$\$\eTT%%A$%A%A%A&%A%A$\$\$\r)rr& ChunkedArray chunked_arraychunks)arrayrrrs ``rrz)_wrap_for_chunked_arrays..wrappersg eR_ - - 0#$\$\$\$\$\$\u|$\$\$\]] ]4////// /rr )rrs` r_wrap_for_chunked_arraysrs#00000 Nrpa_typec.t|tjrtd|DSt|tjtjtjfrt|jSt|tj rdSdS)zE Check (recursively) if a pyarrow type is an extension type. c3>K|]}t|jVdSr%)_is_extension_typetyper?rs rrAz%_is_extension_type..%s-GGe%ej11GGGGGGrTF) rr& StructTypeanyListTypeFixedSizeListType LargeListTyper value_type ExtensionType)rs rrr s'2=))GGwGGGGGG Gbk2+?AQR S S!'"4555 GR- . .turarraysct|}d|D}|stdt|dkr0t|}td|dd|d|dj}d|D}t |st j|Sdfd |S) aImproved version of pa.concat_arrays It supports concatenating pa.ExtensionArray objects by concatenating the underlying storages. Args: arrays (List[pa.Array]): List of arrays to contatenate Raises: pa.ArrowInvalid: if the arrow array concatenation fails ValueError: if the list of arrays is empty TypeError: if the arrays to be concatenated have different types Returns: array (:obj:`pyarrow.Array`): the concatenated array ch|] }|j Sr )rr?rs r zarray_concat..?s222%5:222rz)Couldn't concatenate empty list of arraysr z1Couldn't concatenate arrays with different types rz and c`g|]+}t|tjr|jn|fD]}|,Sr )rr&rr)r?r[rs rrlz array_concat..HsB p p p*SRTRaBbBb4nCJJilhn p p%e p p p prc|d}|}|ddD]V}tj||d}tj|dd|d}tj||g}W|S)Nrr r])pcsubtractaddr& concat_arrays)offsetsrconcatenated_offsetss r_offsets_concatz%array_concat.._offsets_concatMs~%abbk T TF[33FVF122J(._concat_arrays..Ys8[8[8[58[8[8[rc@g|]fdDS)cDg|]}|jSr )rr)r?rrs rrlzCarray_concat.._concat_arrays...\s' M M MUUZ!8!8 M M Mrr )r?r_concat_arraysrs @rrlz8array_concat.._concat_arrays..\s:gggSX M M M Mf M M MNNgggrc6g|]}|Sr is_nullrs rrlz8array_concat.._concat_arrays..^s &K&K&K5u}}&K&K&Krrrc3,K|]}|jdkVdSr) null_countrs rrAz7array_concat.._concat_arrays..as*<=10.0.0` to avoid this behavior. More info: https://github.com/huggingface/datasets/issues/3676.cg|] }|j Sr rrs rrlz8array_concat.._concat_arrays..hs(K(K(K5(K(K(Krcg|] }|j Sr valuesrs rrlz8array_concat.._concat_arrays..is'I'I'I 'I'I'Irc6g|]}|Sr rrs rrlz8array_concat.._concat_arrays..js .S.S.S5u}}.S.S.Srrcg|] }|j Sr rrs rrlz8array_concat.._concat_arrays..ms C C C5 C C Crcg|] }|j Sr rrs rrlz8array_concat.._concat_arrays..nAAA AAArcg|] }|j Sr rrs rrlz8array_concat.._concat_arrays..rr!r)rrr&PyExtensionType wrap_arraytypes is_struct StructArrayr%rris_listrrPYARROW_VERSIONmajorwarningswarn ListArrayis_fixed_size_listFixedSizeListArray list_size)r array_typerr s` rrz$array_concat.._concat_arraysVsiAY^ j""4 5 5 ((8[8[TZ8[8[8[)\)\]] ] X   + + >--ggggg\fgggJ''%&K&KF&K&K&KLL.  X  j ) ) <& ! !!rTrc tt|ttjrjt|tjr|Sj|krStj jrtj |rod|DdjDkrPfd|D}tj |t|Sntj jrgtj |rh|jt%zt%jkr9tj j|j|jSnmtj |rjdkr{t.jjdkrt5jd|d nMtj jj|j Stj jj|jSntj jrptj |r9tj j|j|jStj |rtjt?t%d ztj }jdkrvt.jjdkrt5jd|d nHtj |j|j Stj |j|jSn|stj !|retj "js$tj #jrtId jd |d|tj |r>tj jstId jd |%|StIdjd|)a_Improved version of `pa.Array.cast` It supports casting `pa.StructArray` objects to re-order the fields. It also let you control certain aspects of the casting, e.g. whether to disable numbers (`floats` or `ints`) to strings. Args: array (`pa.Array`): PyArrow array to cast pa_type (`pa.DataType`): Target PyArrow type allow_number_to_str (`bool`, defaults to `True`): Whether to allow casting numbers to strings. Defaults to `True`. Raises: `pa.ArrowInvalidError`: if the arrow data casting fails `TypeError`: if the target type is not supported according, e.g. - if a field is missing - if casting from numbers to strings and `allow_number_to_str` is `False` Returns: `List[pyarrow.Array]`: the casted array allow_number_to_strch|] }|j Sr rrs rrzarray_cast..s,M,M,MEUZ,M,M,Mrch|] }|j Sr rrs rrzarray_cast..sQuQuQuafRWR\QuQuQurcbg|]+}|j|j,Sr )rrr)r?r_crs rrlzarray_cast..s5SSS%bbUZ00%*==SSSrrrrVNone values are converted to empty lists in `pyarrow<10.0.0` when converting array to s. Install `pyarrow>=10.0.0` to avoid this behavior. More info: https://github.com/huggingface/datasets/issues/3676.rr zCouldn't cast array of type z to z% since allow_number_to_str is set to Couldn't cast array of type  to )&r array_castrr&ExtensionArrayr rr$rr%r&r'r%rrr(r.r0r`rr/rrrr)r*r+r,r-rrrint32 is_string is_floating is_integerrlr)rrr6r offsets_arrr:s` @rr?r?zs<6 1D E E EB%*++ '2+,,7#!!%((( w     EJ ' '3# 8  g & & b,M,MW,M,M,MQuQujojtQuQuQu,u,uSSSSS7SSSF>--fT']]QVQ^Q^Q`Q`-aa a   %* % %/# 8 & &w / / a 3u::-U\1B1BBB,88Bu|W%788%C X  g & & a!##)/"44Mnqxnnn<33 rr%,8J'K'KRWR_R_RaRa4<++EM22elGL^;_;_`` ` a  $ $UZ 0 0# 8 & &w / / _(445<!344! X  g & & _(5Ua#8#8"(**EEK!##)/"44Mnqxnnn<33#RR g6H%I%IPUP]P]P_P_4<++KEL'J\9]9]^^ ^ _$ ""7++ %%ej11 68X5H5H5T5T  CuzCCwCCnACC  8  G $ $ VRX-=-=ej-I-I VT5:TT7TTUU Uzz'""" OEJOOgOO P PPrfeaturerc ddlmm}tt|t t jrjtdr St j j rt r@t jtr&fdjDt trdj Dt#krbfdD}t j|t)Snt j j rLt t(rjd }|j jj krSjd krgt2jjd krt9jd d n9t jj| St jj|St r_j dkrhj tCztCjkr9t j"jjj Sndjj}|j jj krSjd kr{t2jjd krt9jd d nMt jjjj St jjjjSnvt j #j rQt t(rjd kr|t2jjd krt9jddnNt jjjd  St jjjd St rjj dkrgj tCztCjkr9t j"jjj Snt j$tKtCdzt j&}jd krvt2jjd krt9jddnHt j|jj St j|jjSt j j rtO||St tt(tPfstO|StSdj d)aCast an array to the arrow type that corresponds to the requested feature type. For custom features like [`Audio`] or [`Image`], it takes into account the "cast_storage" methods they defined to enable casting from other arrow types. Args: array (`pa.Array`): The PyArrow array to cast. feature (`datasets.features.FeatureType`): The target feature type. allow_number_to_str (`bool`, defaults to `True`): Whether to allow casting numbers to strings. Defaults to `True`. Raises: `pa.ArrowInvalidError`: if the arrow data casting fails `TypeError`: if the target type is not supported according, e.g. - if a field is missing - if casting from numbers to strings and `allow_number_to_str` is `False` Returns: array (`pyarrow.Array`): the casted array r )Sequenceget_nested_typer5 cast_storagec<i|]\}}||jSrrr?r subfeaturerHrFs rrz)cast_array_to_feature..=FVdJhhz'.AAArch|] }|j Sr rrs rrz(cast_array_to_feature..s)M)M)M%*)M)M)MrcTg|]$\}}||%Sr r)r?rrNr:rs rrlz)cast_array_to_feature..6```=10.0.0` to avoid this behavior. More info: https://github.com/huggingface/datasets/issues/3676. This will raise an error in a future major version of `datasets`r=r>)*features.featuresrHrIrcast_array_to_featurerr&r@r rrJr%r&rrFrrQsetr'r%rrr(rrrr)r*r+r,r-rrr`r/r.rrrAr?rrl) rrFr6rIr casted_valuesrErHr:s `` @@rrWrWs2=<<<<<<< &--fDMMPUP]P]P_P_-`` `   %* % %<` gt $ $ fBu|WQZ88M!U\%666 #a''-3b88 ru|rrr "|77 }[`[h[h[j[j7kkk|// }MMM  * * f~"">CJJ.#el2C2CCC0<CJJ.#el2C2CCC0< ? ?U%@STTTT OEJOOgOO P PPrc ddlmtttjrjtdrStj j rtr@tj tr&fdj DttrbfdD}tj|t#Sntj j rHtt"rjdkr|t*jjdkrt1jd d nNtjjjd StjjjdStr_jd krhjt=zt=jkr9tjjj jSndjj }|j jj krSjdkr{t*jjdkrt1jd d nMtjjjj  Stjjjj Snvtj j rQtt"rjdkr|t*jjdkrt1jd d nNtjjjd StjjjdStrjjd krgjt=zt=jkr9tjjj jSnt j!tEt=dzt j#}jdkrvt*jjdkrt1jd d nHtj|jj  Stj|jj Sttt"tHfsStKd j d)aqEmbed data into an arrays's storage. For custom features like Audio or Image, it takes into account the "embed_storage" methods they defined to enable embedding external data (e.g. an image file) into an other arrow types. Args: array (`pa.Array`): The PyArrow array in which to embed data. feature (`datasets.features.FeatureType`): Array features. Raises: `TypeError`: if the target type is not supported according, e.g. - if a field is missing Returns: array (`pyarrow.Array`): the casted array r )rH embed_storagec<i|]\}}||jSrLrrMs rrz'embed_array_storage..brOrcTg|]$\}}||%Sr rR)r?rrN_ers rrlz'embed_array_storage..frSrrTrrzKNone values are converted to empty lists when embedding array storage with rUrr]zCouldn't embed array of type z with )&rrHembed_array_storagerr&r@r rr[r%r&rrFrrQr'r%rrr(rrr)r*r+r,r-rrrr`r/r.rrrArrl)rrFrrYrErHr^s`` @@rr_r_AsO,#""""" B%*++ w((C`$$U+++   EJ ' 'A` gx ( ( Z-N-N ZaZiZoZoZqZqG gt $ $ a`````PWP]P]P_P_```F>--fDMMPUP]P]P_P_-`` ` a   %* % %8` gt $ $ f!##)/"44Mdfmddd<33EM22elT[\]T^C_C_fkfsfsfufu3vvv<++EM22elGTUJ;W;WXX X  * * f~"">CJJ.#el2C2CCC0<CJJ.#el2C2CCC0<! because column names don't matchcBg|]\}}t||Sr rWr?rrFr3s rrlz*cast_table_to_features.., ` ` `mdG#E$K99 ` ` `rr) sortedrrr<rQr&rr%r)r3rrs` rcast_table_to_featuresrhse !!VH%5%555k5<kkxkkklll ` ` ` `x~~O_O_ ` ` `F 8  x/D  E EErr<c2ddlm}|j|}tjt|krt djd|dfd|D}tj ||S)a0Cast a table to the arrow schema. Different from `cast_table_to_features`, this method can preserve nullability. Args: table (`pa.Table`): PyArrow table to cast. features ([`Features`]): Target features. Returns: `pa.Table`: the casted table r rrar>rbcBg|]\}}t||Sr rdres rrlz(cast_table_to_schema..rfrr) rrrrgrrr<rQr&rr%)r3r<rrrs` rcast_table_to_schemarks#""""")x)&11H e !!VH%5%555k5<kkxkkklll ` ` ` `x~~O_O_ ` ` `F 8  v  6 66rcddlm}m|jj}fd|D}t j||j S)zEmbed external data into a table's storage. Args: table (`pyarrow.Table`): PyArrow table in which to embed data. Returns: table (`pyarrow.Table`): the table with embedded data r )rrequire_storage_embedchg|].\}}|rt||n|/Sr )r_)r?rrFrmr3s rrlz'embed_table_storage..sY D'6K5J75S5SdE$K111Y^_cYdrr) rVrrmrr<rQr&rr%r)r3rrrrms` @rembed_table_storagerosCBBBBBBB)x)%,77H%^^--F 8  x/D  E EErc|j|krt||S|jj|jkr||jS|S)aGImproved version of `pa.Table.cast`. It supports casting to feature types stored in the schema metadata. Args: table (`pyarrow.Table`): PyArrow table to cast. schema (`pyarrow.Schema`): Target PyArrow schema. Returns: table (`pyarrow.Table`): the casted table )r<rkmetadatarrs rr3r3sN |v#E6222  &/ 1 1,,V_=== rc ddlm}|j|j}t d|Dr g}g}|jD]|j}|j}tj j rvt|dr| |krN|| |fdj D|||jtj||}n| }| d | fd |jD | jjS) aImproved version of `pa.Table.flatten`. It behaves as `pa.Table.flatten` in a sense it does 1-step flatten of the columns with a struct type into one column per struct field, but updates the metadata and skips decodable features unless the `decode` attribute of these features is set to False. Args: table (`pa.Table`): PyArrow table to flatten. Returns: `Table`: the flattened table r rc3fK|],}t|do||kV-dS)rN)rr)r?rNs rrAz table_flatten.. sC u uU_7:y ) ) Pj.@.@.B.Bj.P u u u u u urrc2g|]}jd|jS)r_r)r?rrs rrlz!table_flatten.. s+)e)e)ehUZ*I*I(-*I*I)e)e)er)r) max_depthc"i|] }|| Sr r )r? column_name flat_featuress rrz!table_flatten.. s qqq+k=+Eqqqr)rrrr<rrrrr&r%r&rrrrrUrr%rrrrq) r3rr flat_arraysflat_column_namesrrN flat_tablerrys @@rr0r0s#""""")x)%,77H u uckcrcrctct u u uuu% \ 5 5ELL,,E!%*-Jx!!%*-- 5J 22 56@6H6H6J6Jj6X6X""5==??333!(()e)e)e)eZ_Zd)e)e)effff""5)))!((4444X)) #*  ]]__ $$q$11MHqqqqYcYpqqqrrM  - -m.H.Q R RRrfunctioncddlm}m|j|j}fd|D]\}}|||dS)zVisit all arrays in a table and apply a function to them. Args: table (`pyarrow.Table`): PyArrow table to visit. function (`Callable[[pa.Array], None]`): Function to apply to each array. r )rrHc>t|tjr|jD]}|dSt|tjr|j}|tj|jrtdstr@tj tr&fdj D D]$\}}| ||%dStj|jrVttr|jddStr|jj dSdSdS)NrJc<i|]\}}||jSrLrrMs rrz1table_visitor.._visit..: s@,D*hhz'.IIIrr)rr&rrr@r r%r&rrrFrrQrr(rr)rrFrrrNrH_visitr}s ` rrztable_visitor.._visit0 s eR_ - - : ' 'ug&&&& ' '%!233 &  HUG $ $ $x!!%*-- :gg~6V6V :gx00ZQU5V5V070E0E0G0GG)0 ::$D*F5;;t,,j9999::!!%*-- :gt,,:F5<4444422:F5<99999  : :::rN)rrrHrr<rQ)r3r}rrrrFrHrs ` @@r table_visitorr# s-,,,,,,,)x)%,77H:::::::,"))%% guT{G$$$$%%rF batch_sizec#Kg}d}||D].}t|dkr|t|z|kr(|||t|z }U|t|z|kr;||tj|Vg}d}||z }||d|tj|V||t||z g}t||z }0|s%|r%tj|VdSdSdS)aMIterate over sub-tables of size `batch_size`. Args: table (`pyarrow.Table`): PyArrow table to iterate over. batch_size (`int`): Size of each sub-table to yield. drop_last_batch (`bool`, defaults to `False`): Drop the last batch if it is smaller than `batch_size`. rrN)rr`rUr&rrr})r3rdrop_last_batch chunks_bufferchunks_buffer_sizercropped_chunk_lengths r table_iterrJ sMz::CC u::??  #e** ,z 9 9   ' ' ' #e** ,   #e** , : :   ' ' '('' 66 6 6 6M!"  #-0B#B  Q0D!E!E F F F('' 66 6 6 6"[[)=s5zzL`?`aabM!$U.B!B   3}3h##M22222223333rri)T)F)HrSrrr+ functoolsr itertoolsrtypingrrrrr r r r numpyrrpyarrowr&pyarrow.computecomputerrr utils.loggingrrVrrrrr!rErr.rFr4r8rrJrrZrdrfrrrrerHrgrirwrrDataTypeboolrArrayr3r?rWr_rhSchemarkror3r0rrr rrrs" [[[[[[[[[[[[[[[[[[[[%%%%%%988888888 H  srxryRX32866C6C6666tYtCyYSYSYYYY4.C.C.C.C.C.C.C.Cbv$v$v$v$v$ v$v$v$r        XAXAXAXAXAJXAXAXAx sE4 RYRYRYRYRY RYRYRYrg3ZjAQSWX\]gXhSijjP1P1P1P1P1P1P1P1f==$u+=S=====.%DI*         I"bhI"I"I"I"XUQUQbhUQUQUQUQUQpjQjQjQMjQjQjQjQZaTrxaT-aTaTaTaTHF"(FjFFFF$77")7777*FrxFFFF,bh ,'S'S'S'S'ST$%$%Xrxj$6F-G$%$%$%$%N 3 3e 3 3QSQYHZ 3 3 3 3 3 3r