# Licensed to the Apache Software Foundation (ASF) under one

# or more contributor license agreements. See the NOTICE file

# distributed with this work for additional information

# regarding copyright ownership. The ASF licenses this file

# to you under the Apache License, Version 2.0 (the

# "License"); you may not use this file except in compliance

# with the License. You may obtain a copy of the License at

#

# http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing,

# software distributed under the License is distributed on an

# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY

# KIND, either express or implied. See the License for the

# specific language governing permissions and limitations

# under the License.

from cpython.pycapsule cimport PyCapsule_CheckExact, PyCapsule_GetPointer, PyCapsule_New

import warnings

from cython import sizeof

cdef class ChunkedArray(_PandasConvertible):

"""

An array-like composed from a (possibly empty) collection of pyarrow.Arrays

Warnings

--------

Do not call this class's constructor directly.

Examples

--------

To construct a ChunkedArray object use :func:`pyarrow.chunked_array`:

>>> import pyarrow as pa

>>> pa.chunked_array([], type=pa.int8())

<pyarrow.lib.ChunkedArray object at ...>

[

...

]

>>> pa.chunked_array([[2, 2, 4], [4, 5, 100]])

<pyarrow.lib.ChunkedArray object at ...>

[

[

2,

2,

4

],

[

4,

5,

100

]

]

>>> isinstance(pa.chunked_array([[2, 2, 4], [4, 5, 100]]), pa.ChunkedArray)

True

"""

def __cinit__(self):

self.chunked_array = NULL

self._init_is_cpu = False

def __init__(self):

raise TypeError("Do not call ChunkedArray's constructor directly, use "

"`chunked_array` function instead.")

cdef void init(self, const shared_ptr[CChunkedArray]& chunked_array):

self.sp_chunked_array = chunked_array

self.chunked_array = chunked_array.get()

def __reduce__(self):

self._assert_cpu()

return chunked_array, (self.chunks, self.type)

@property

def data(self):

import warnings

warnings.warn("Calling .data on ChunkedArray is provided for "

"compatibility after Column was removed, simply drop "

"this attribute", FutureWarning)

return self

@property

def type(self):

"""

Return data type of a ChunkedArray.

Examples

--------

>>> import pyarrow as pa

>>> n_legs = pa.chunked_array([[2, 2, 4], [4, 5, 100]])

>>> n_legs.type

DataType(int64)

"""

return pyarrow_wrap_data_type(self.sp_chunked_array.get().type())

def length(self):

"""

Return length of a ChunkedArray.

Examples

--------

>>> import pyarrow as pa

>>> n_legs = pa.chunked_array([[2, 2, 4], [4, 5, 100]])

>>> n_legs.length()

6

"""

return self.chunked_array.length()

def __len__(self):

return self.length()

def __repr__(self):

type_format = object.__repr__(self)

return f"{type_format}\n{self}"

def to_string(self, *, int indent=0, int window=5, int container_window=2,

c_bool skip_new_lines=False, int element_size_limit=100):

"""

Render a "pretty-printed" string representation of the ChunkedArray

Parameters

----------

indent : int

How much to indent right the content of the array,

by default ``0``.

window : int

How many items to preview within each chunk at the begin and end

of the chunk when the chunk is bigger than the window.

The other elements will be ellipsed.

container_window : int

How many chunks to preview at the begin and end

of the array when the array is bigger than the window.

The other elements will be ellipsed.

This setting also applies to list columns.

skip_new_lines : bool

If the array should be rendered as a single line of text

or if each element should be on its own line.

element_size_limit : int, default 100

Maximum number of characters of a single element before it is truncated.

Examples

--------

>>> import pyarrow as pa

>>> n_legs = pa.chunked_array([[2, 2, 4], [4, 5, 100]])

>>> n_legs.to_string(skip_new_lines=True)

'[[2,2,4],[4,5,100]]'

"""

cdef:

c_string result

PrettyPrintOptions options

with nogil:

options = PrettyPrintOptions(indent, window)

options.skip_new_lines = skip_new_lines

options.container_window = container_window

options.element_size_limit = element_size_limit

check_status(

PrettyPrint(

deref(self.chunked_array),

options,

&result

)

)

return frombytes(result, safe=True)

def format(self, **kwargs):

"""

DEPRECATED, use pyarrow.ChunkedArray.to_string

Parameters

----------

**kwargs : dict

Returns

-------

str

"""

import warnings

warnings.warn('ChunkedArray.format is deprecated, '

'use ChunkedArray.to_string')

return self.to_string(**kwargs)

def __str__(self):

return self.to_string()

def validate(self, *, full=False):

"""

Perform validation checks. An exception is raised if validation fails.

By default only cheap validation checks are run. Pass `full=True`

for thorough validation checks (potentially O(n)).

Parameters

----------

full : bool, default False

If True, run expensive checks, otherwise cheap checks only.

Raises

------

ArrowInvalid

"""

if full:

self._assert_cpu()

with nogil:

check_status(self.sp_chunked_array.get().ValidateFull())

else:

with nogil:

check_status(self.sp_chunked_array.get().Validate())

@property

def null_count(self):

"""

Number of null entries

Returns

-------

int

Examples

--------

>>> import pyarrow as pa

>>> n_legs = pa.chunked_array([[2, 2, 4], [4, None, 100]])

>>> n_legs.null_count

1

"""

self._assert_cpu()

return self.chunked_array.null_count()

@property

def nbytes(self):

"""

Total number of bytes consumed by the elements of the chunked array.

In other words, the sum of bytes from all buffer ranges referenced.

Unlike `get_total_buffer_size` this method will account for array

offsets.

If buffers are shared between arrays then the shared

portion will only be counted multiple times.

The dictionary of dictionary arrays will always be counted in their

entirety even if the array only references a portion of the dictionary.

Examples

--------

>>> import pyarrow as pa

>>> n_legs = pa.chunked_array([[2, 2, 4], [4, None, 100]])

>>> n_legs.nbytes

49

"""

self._assert_cpu()

cdef:

CResult[int64_t] c_res_buffer

with nogil:

c_res_buffer = ReferencedBufferSize(deref(self.chunked_array))

size = GetResultValue(c_res_buffer)

return size

def get_total_buffer_size(self):

"""

The sum of bytes in each buffer referenced by the chunked array.

An array may only reference a portion of a buffer.

This method will overestimate in this case and return the

byte size of the entire buffer.

If a buffer is referenced multiple times then it will

only be counted once.

Examples

--------

>>> import pyarrow as pa

>>> n_legs = pa.chunked_array([[2, 2, 4], [4, None, 100]])

>>> n_legs.get_total_buffer_size()

49

"""

self._assert_cpu()

cdef:

int64_t total_buffer_size

total_buffer_size = TotalBufferSize(deref(self.chunked_array))

return total_buffer_size

def __sizeof__(self):

return super(ChunkedArray, self).__sizeof__() + self.nbytes

def __iter__(self):

for chunk in self.iterchunks():

for item in chunk:

yield item

def __getitem__(self, key):

"""

Slice or return value at given index

Parameters

----------

key : integer or slice

Slices with step not equal to 1 (or None) will produce a copy

rather than a zero-copy view

Returns

-------

value : Scalar (index) or ChunkedArray (slice)

"""

self._assert_cpu()

if isinstance(key, slice):

return _normalize_slice(self, key)

return self.getitem(_normalize_index(key, self.chunked_array.length()))

cdef getitem(self, int64_t i):

self._assert_cpu()

return Scalar.wrap(GetResultValue(self.chunked_array.GetScalar(i)))

def is_null(self, *, nan_is_null=False):

"""

Return boolean array indicating the null values.

Parameters

----------

nan_is_null : bool (optional, default False)

Whether floating-point NaN values should also be considered null.

Returns

-------

array : boolean Array or ChunkedArray

Examples

--------

>>> import pyarrow as pa

>>> n_legs = pa.chunked_array([[2, 2, 4], [4, None, 100]])

>>> n_legs.is_null()

<pyarrow.lib.ChunkedArray object at ...>

[

[

false,

false,

false,

false,

true,

false

]

]

"""

self._assert_cpu()

options = _pc().NullOptions(nan_is_null=nan_is_null)

return _pc().call_function('is_null', [self], options)

def is_nan(self):

"""

Return boolean array indicating the NaN values.

Examples

--------

>>> import pyarrow as pa

>>> import numpy as np

>>> arr = pa.chunked_array([[2, np.nan, 4], [4, None, 100]])

>>> arr.is_nan()

<pyarrow.lib.ChunkedArray object at ...>

[

[

false,

true,

false,

false,

null,

false

]

]

"""

self._assert_cpu()

return _pc().is_nan(self)

def is_valid(self):

"""

Return boolean array indicating the non-null values.

Examples

--------

>>> import pyarrow as pa

>>> n_legs = pa.chunked_array([[2, 2, 4], [4, None, 100]])

>>> n_legs.is_valid()

<pyarrow.lib.ChunkedArray object at ...>

[

[

true,

true,

true

],

[

true,

false,

true

]

]

"""

self._assert_cpu()

return _pc().is_valid(self)

def __eq__(self, other):

try:

return self.equals(other)

except TypeError:

return NotImplemented

def fill_null(self, fill_value):

"""

Replace each null element in values with fill_value.

See :func:`pyarrow.compute.fill_null` for full usage.

Parameters

----------

fill_value : any

The replacement value for null entries.

Returns

-------

result : Array or ChunkedArray

A new array with nulls replaced by the given value.

Examples

--------

>>> import pyarrow as pa

>>> fill_value = pa.scalar(5, type=pa.int8())

>>> n_legs = pa.chunked_array([[2, 2, 4], [4, None, 100]])

>>> n_legs.fill_null(fill_value)

<pyarrow.lib.ChunkedArray object at ...>

[

[

2,

2,

4,

4,

5,

100

]

]

"""

self._assert_cpu()

return _pc().fill_null(self, fill_value)

def equals(self, ChunkedArray other):

"""

Return whether the contents of two chunked arrays are equal.

Parameters

----------

other : pyarrow.ChunkedArray

Chunked array to compare against.

Returns

-------

are_equal : bool

Examples

--------

>>> import pyarrow as pa

>>> n_legs = pa.chunked_array([[2, 2, 4], [4, 5, 100]])

>>> animals = pa.chunked_array((

... ["Flamingo", "Parrot", "Dog"],

... ["Horse", "Brittle stars", "Centipede"]

... ))

>>> n_legs.equals(n_legs)

True

>>> n_legs.equals(animals)

False

"""

self._assert_cpu()

if other is None:

return False

cdef:

CChunkedArray* this_arr = self.chunked_array

CChunkedArray* other_arr = other.chunked_array

c_bool result

with nogil:

result = this_arr.Equals(deref(other_arr))

return result

def _to_pandas(self, options, types_mapper=None, **kwargs):

self._assert_cpu()

return _array_like_to_pandas(self, options, types_mapper=types_mapper)

def to_numpy(self, zero_copy_only=False):

"""

Return a NumPy copy of this array (experimental).

Parameters

----------

zero_copy_only : bool, default False

Introduced for signature consistence with pyarrow.Array.to_numpy.

This must be False here since NumPy arrays' buffer must be contiguous.

Returns

-------

array : numpy.ndarray

Examples

--------

>>> import pyarrow as pa

>>> n_legs = pa.chunked_array([[2, 2, 4], [4, 5, 100]])

>>> n_legs.to_numpy()

array([ 2, 2, 4, 4, 5, 100])

"""

self._assert_cpu()

if np is None:

raise ImportError(

"Cannot return a numpy.ndarray if NumPy is not present")

if zero_copy_only:

raise ValueError(

"zero_copy_only must be False for pyarrow.ChunkedArray.to_numpy"

)

cdef:

PyObject* out

PandasOptions c_options

object values

c_options.to_numpy = True

with nogil:

check_status(

ConvertChunkedArrayToPandas(

c_options,

self.sp_chunked_array,

self,

&out

)

)

# wrap_array_output uses pandas to convert to Categorical, here

# always convert to numpy array

values = PyObject_to_object(out)

if isinstance(values, dict):

values = np.take(values['dictionary'], values['indices'])

return values

def __array__(self, dtype=None, copy=None):

self._assert_cpu()

if copy is False:

raise ValueError(

"Unable to avoid a copy while creating a numpy array as requested "

"(converting a pyarrow.ChunkedArray always results in a copy).\n"

"If using `np.array(obj, copy=False)` replace it with "

"`np.asarray(obj)` to allow a copy when needed"

)

# 'copy' can further be ignored because to_numpy() already returns a copy

values = self.to_numpy()

if dtype is None:

return values

return values.astype(dtype, copy=False)

def cast(self, object target_type=None, safe=None, options=None):

"""

Cast array values to another data type

See :func:`pyarrow.compute.cast` for usage.

Parameters

----------

target_type : DataType, None

Type to cast array to.

safe : boolean, default True

Whether to check for conversion errors such as overflow.

options : CastOptions, default None

Additional checks pass by CastOptions

Returns

-------

cast : Array or ChunkedArray

Examples

--------

>>> import pyarrow as pa

>>> n_legs = pa.chunked_array([[2, 2, 4], [4, 5, 100]])

>>> n_legs.type

DataType(int64)

Change the data type of an array:

>>> n_legs_seconds = n_legs.cast(pa.duration('s'))

>>> n_legs_seconds.type

DurationType(duration[s])

"""

self._assert_cpu()

return _pc().cast(self, target_type, safe=safe, options=options)

def dictionary_encode(self, null_encoding='mask'):

"""

Compute dictionary-encoded representation of array.

See :func:`pyarrow.compute.dictionary_encode` for full usage.

Parameters

----------

null_encoding : str, default "mask"

How to handle null entries.

Returns

-------

encoded : ChunkedArray

A dictionary-encoded version of this array.

Examples

--------

>>> import pyarrow as pa

>>> animals = pa.chunked_array((

... ["Flamingo", "Parrot", "Dog"],

... ["Horse", "Brittle stars", "Centipede"]

... ))

>>> animals.dictionary_encode()

<pyarrow.lib.ChunkedArray object at ...>

[

...

-- dictionary:

[

"Flamingo",

"Parrot",

"Dog",

"Horse",

"Brittle stars",

"Centipede"

]

-- indices:

[

0,

1,

2

],

...

-- dictionary:

[

"Flamingo",

"Parrot",

"Dog",

"Horse",

"Brittle stars",

"Centipede"

]

-- indices:

[

3,

4,

5

]

]

"""

self._assert_cpu()

options = _pc().DictionaryEncodeOptions(null_encoding)

return _pc().call_function('dictionary_encode', [self], options)

def flatten(self, MemoryPool memory_pool=None):

"""

Flatten this ChunkedArray. If it has a struct type, the column is

flattened into one array per struct field.

Parameters

----------

memory_pool : MemoryPool, default None

For memory allocations, if required, otherwise use default pool

Returns

-------

result : list of ChunkedArray

Examples

--------

>>> import pyarrow as pa

>>> n_legs = pa.chunked_array([[2, 2, 4], [4, 5, 100]])

>>> c_arr = pa.chunked_array(n_legs.value_counts())

>>> c_arr

<pyarrow.lib.ChunkedArray object at ...>

[

-- is_valid: all not null

-- child 0 type: int64

[

2,

4,

5,

100

]

-- child 1 type: int64

[

2,

2,

1,

1

]

]

>>> c_arr.flatten()

[<pyarrow.lib.ChunkedArray object at ...>

[

[

2,

4,

5,

100

]

], <pyarrow.lib.ChunkedArray object at ...>

[

[

2,

2,

1,

1

]

]]

>>> c_arr.type

StructType(struct<values: int64, counts: int64>)

>>> n_legs.type

DataType(int64)

"""

self._assert_cpu()

cdef:

vector[shared_ptr[CChunkedArray]] flattened

CMemoryPool* pool = maybe_unbox_memory_pool(memory_pool)

with nogil:

flattened = GetResultValue(self.chunked_array.Flatten(pool))

return [pyarrow_wrap_chunked_array(col) for col in flattened]

def combine_chunks(self, MemoryPool memory_pool=None):

"""

Flatten this ChunkedArray into a single non-chunked array.

Parameters

----------

memory_pool : MemoryPool, default None

For memory allocations, if required, otherwise use default pool

Returns

-------

result : Array

Examples

--------

>>> import pyarrow as pa

>>> n_legs = pa.chunked_array([[2, 2, 4], [4, 5, 100]])

>>> n_legs

<pyarrow.lib.ChunkedArray object at ...>

[

[

2,

2,

4

],

[

4,

5,

100

]

]

>>> n_legs.combine_chunks()

<pyarrow.lib.Int64Array object at ...>

[

2,

2,

4,

4,

5,

100

]

"""

self._assert_cpu()

if self.num_chunks == 0:

return array([], type=self.type)

else:

return concat_arrays(self.chunks)

def unique(self):

"""

Compute distinct elements in array

Returns

-------

pyarrow.Array

Examples

--------

>>> import pyarrow as pa

>>> n_legs = pa.chunked_array([[2, 2, 4], [4, 5, 100]])

>>> n_legs

<pyarrow.lib.ChunkedArray object at ...>

[

[

2,

2,

4

],

[

4,

5,

100

]

]

>>> n_legs.unique()

<pyarrow.lib.Int64Array object at ...>

[

2,

4,

5,

100

]

"""

self._assert_cpu()

return _pc().call_function('unique', [self])

def value_counts(self):

"""

Compute counts of unique elements in array.

Returns

-------

An array of <input type "Values", int64_t "Counts"> structs

Examples

--------

>>> import pyarrow as pa

>>> n_legs = pa.chunked_array([[2, 2, 4], [4, 5, 100]])

>>> n_legs

<pyarrow.lib.ChunkedArray object at ...>

[

[

2,

2,

4

],

[

4,

5,

100

]

]

>>> n_legs.value_counts()

<pyarrow.lib.StructArray object at ...>

-- is_valid: all not null

-- child 0 type: int64

[

2,

4,

5,

100

]

-- child 1 type: int64

[

2,

2,

1,

1

]

"""

self._assert_cpu()

return _pc().call_function('value_counts', [self])

def slice(self, offset=0, length=None):

"""

Compute zero-copy slice of this ChunkedArray

Parameters

----------

offset : int, default 0

Offset from start of array to slice

length : int, default None

Length of slice (default is until end of batch starting from

offset)

Returns

-------

sliced : ChunkedArray

Examples

--------

>>> import pyarrow as pa

>>> n_legs = pa.chunked_array([[2, 2, 4], [4, 5, 100]])

>>> n_legs

<pyarrow.lib.ChunkedArray object at ...>

[

[

2,

2,

4

],

[

4,

5,

100

]

]

>>> n_legs.slice(2,2)

<pyarrow.lib.ChunkedArray object at ...>

[

[

4

],

[

4

]

]

"""

cdef shared_ptr[CChunkedArray] result

if offset < 0:

raise IndexError('Offset must be non-negative')

offset = min(len(self), offset)

if length is None:

result = self.chunked_array.Slice(offset)

else:

result = self.chunked_array.Slice(offset, length)

return pyarrow_wrap_chunked_array(result)

def filter(self, mask, object null_selection_behavior="drop"):

"""

Select values from the chunked array.

See :func:`pyarrow.compute.filter` for full usage.

Parameters

----------

mask : Array or array-like

The boolean mask to filter the chunked array with.

null_selection_behavior : str, default "drop"

How nulls in the mask should be handled.

Returns

-------

filtered : Array or ChunkedArray

An array of the same type, with only the elements selected by

the boolean mask.

Examples

--------

>>> import pyarrow as pa

>>> n_legs = pa.chunked_array([[2, 2, 4], [4, 5, 100]])

>>> n_legs

<pyarrow.lib.ChunkedArray object at ...>

[

[

2,

2,

4

],

[

4,

5,

100

]

]

>>> mask = pa.array([True, False, None, True, False, True])

>>> n_legs.filter(mask)

<pyarrow.lib.ChunkedArray object at ...>

[

[

2

],

[

4,

100

]

]

>>> n_legs.filter(mask, null_selection_behavior="emit_null")

<pyarrow.lib.ChunkedArray object at ...>

[

[

2,

null

],

[

4,

100

]

]

"""

self._assert_cpu()

return _pc().filter(self, mask, null_selection_behavior)

def index(self, value, start=None, end=None, *, memory_pool=None):

"""

Find the first index of a value.

See :func:`pyarrow.compute.index` for full usage.

Parameters

----------

value : Scalar or object

The value to look for in the array.

start : int, optional