# Licensed to the Apache Software Foundation (ASF) under one

# or more contributor license agreements. See the NOTICE file

# distributed with this work for additional information

# regarding copyright ownership. The ASF licenses this file

# to you under the Apache License, Version 2.0 (the

# "License"); you may not use this file except in compliance

# with the License. You may obtain a copy of the License at

#

# http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing,

# software distributed under the License is distributed on an

# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY

# KIND, either express or implied. See the License for the

# specific language governing permissions and limitations

# under the License.

from collections.abc import Iterable

import datetime

import decimal

import hypothesis as h

import hypothesis.strategies as st

import itertools

import pickle

import pytest

import struct

import sys

import weakref

import numpy as np

try:

import pickle5

except ImportError:

pickle5 = None

import pyarrow as pa

import pyarrow.tests.strategies as past

def test_total_bytes_allocated():

assert pa.total_allocated_bytes() == 0

def test_weakref():

arr = pa.array([1, 2, 3])

wr = weakref.ref(arr)

assert wr() is not None

del arr

assert wr() is None

def test_getitem_NULL():

arr = pa.array([1, None, 2])

assert arr[1].as_py() is None

assert arr[1].is_valid is False

assert isinstance(arr[1], pa.Int64Scalar)

def test_constructor_raises():

# This could happen by wrong capitalization.

# ARROW-2638: prevent calling extension class constructors directly

with pytest.raises(TypeError):

pa.Array([1, 2])

def test_list_format():

arr = pa.array([[1], None, [2, 3, None]])

result = arr.to_string()

expected = """\

[

[

1

],

null,

[

2,

3,

null

]

]"""

assert result == expected

def test_string_format():

arr = pa.array(['', None, 'foo'])

result = arr.to_string()

expected = """\

[

"",

null,

"foo"

]"""

assert result == expected

def test_long_array_format():

arr = pa.array(range(100))

result = arr.to_string(window=2)

expected = """\

[

0,

1,

...

98,

99

]"""

assert result == expected

def test_indented_string_format():

arr = pa.array(['', None, 'foo'])

result = arr.to_string(indent=1)

expected = '[\n "",\n null,\n "foo"\n]'

assert result == expected

def test_top_level_indented_string_format():

arr = pa.array(['', None, 'foo'])

result = arr.to_string(top_level_indent=1)

expected = ' [\n "",\n null,\n "foo"\n ]'

assert result == expected

def test_binary_format():

arr = pa.array([b'\x00', b'', None, b'\x01foo', b'\x80\xff'])

result = arr.to_string()

expected = """\

[

00,

,

null,

01666F6F,

80FF

]"""

assert result == expected

def test_binary_total_values_length():

arr = pa.array([b'0000', None, b'11111', b'222222', b'3333333'],

type='binary')

large_arr = pa.array([b'0000', None, b'11111', b'222222', b'3333333'],

type='large_binary')

assert arr.total_values_length == 22

assert arr.slice(1, 3).total_values_length == 11

assert large_arr.total_values_length == 22

assert large_arr.slice(1, 3).total_values_length == 11

def test_to_numpy_zero_copy():

arr = pa.array(range(10))

np_arr = arr.to_numpy()

# check for zero copy (both arrays using same memory)

arrow_buf = arr.buffers()[1]

assert arrow_buf.address == np_arr.ctypes.data

arr = None

import gc

gc.collect()

# Ensure base is still valid

assert np_arr.base is not None

expected = np.arange(10)

np.testing.assert_array_equal(np_arr, expected)

def test_to_numpy_unsupported_types():

# ARROW-2871: Some primitive types are not yet supported in to_numpy

bool_arr = pa.array([True, False, True])

with pytest.raises(ValueError):

bool_arr.to_numpy()

result = bool_arr.to_numpy(zero_copy_only=False)

expected = np.array([True, False, True])

np.testing.assert_array_equal(result, expected)

null_arr = pa.array([None, None, None])

with pytest.raises(ValueError):

null_arr.to_numpy()

result = null_arr.to_numpy(zero_copy_only=False)

expected = np.array([None, None, None], dtype=object)

np.testing.assert_array_equal(result, expected)

arr = pa.array([1, 2, None])

with pytest.raises(ValueError, match="with 1 nulls"):

arr.to_numpy()

def test_to_numpy_writable():

arr = pa.array(range(10))

np_arr = arr.to_numpy()

# by default not writable for zero-copy conversion

with pytest.raises(ValueError):

np_arr[0] = 10

np_arr2 = arr.to_numpy(zero_copy_only=False, writable=True)

np_arr2[0] = 10

assert arr[0].as_py() == 0

# when asking for writable, cannot do zero-copy

with pytest.raises(ValueError):

arr.to_numpy(zero_copy_only=True, writable=True)

@pytest.mark.parametrize('unit', ['s', 'ms', 'us', 'ns'])

def test_to_numpy_datetime64(unit):

arr = pa.array([1, 2, 3], pa.timestamp(unit))

expected = np.array([1, 2, 3], dtype="datetime64[{}]".format(unit))

np_arr = arr.to_numpy()

np.testing.assert_array_equal(np_arr, expected)

@pytest.mark.parametrize('unit', ['s', 'ms', 'us', 'ns'])

def test_to_numpy_timedelta64(unit):

arr = pa.array([1, 2, 3], pa.duration(unit))

expected = np.array([1, 2, 3], dtype="timedelta64[{}]".format(unit))

np_arr = arr.to_numpy()

np.testing.assert_array_equal(np_arr, expected)

def test_to_numpy_dictionary():

# ARROW-7591

arr = pa.array(["a", "b", "a"]).dictionary_encode()

expected = np.array(["a", "b", "a"], dtype=object)

np_arr = arr.to_numpy(zero_copy_only=False)

np.testing.assert_array_equal(np_arr, expected)

@pytest.mark.pandas

def test_to_pandas_zero_copy():

import gc

arr = pa.array(range(10))

for i in range(10):

series = arr.to_pandas()

assert sys.getrefcount(series) == 2

series = None # noqa

assert sys.getrefcount(arr) == 2

for i in range(10):

arr = pa.array(range(10))

series = arr.to_pandas()

arr = None

gc.collect()

# Ensure base is still valid

# Because of py.test's assert inspection magic, if you put getrefcount

# on the line being examined, it will be 1 higher than you expect

base_refcount = sys.getrefcount(series.values.base)

assert base_refcount == 2

series.sum()

@pytest.mark.nopandas

@pytest.mark.pandas

def test_asarray():

# ensure this is tested both when pandas is present or not (ARROW-6564)

arr = pa.array(range(4))

# The iterator interface gives back an array of Int64Value's

np_arr = np.asarray([_ for _ in arr])

assert np_arr.tolist() == [0, 1, 2, 3]

assert np_arr.dtype == np.dtype('O')

assert type(np_arr[0]) == pa.lib.Int64Value

# Calling with the arrow array gives back an array with 'int64' dtype

np_arr = np.asarray(arr)

assert np_arr.tolist() == [0, 1, 2, 3]

assert np_arr.dtype == np.dtype('int64')

# An optional type can be specified when calling np.asarray

np_arr = np.asarray(arr, dtype='str')

assert np_arr.tolist() == ['0', '1', '2', '3']

# If PyArrow array has null values, numpy type will be changed as needed

# to support nulls.

arr = pa.array([0, 1, 2, None])

assert arr.type == pa.int64()

np_arr = np.asarray(arr)

elements = np_arr.tolist()

assert elements[:3] == [0., 1., 2.]

assert np.isnan(elements[3])

assert np_arr.dtype == np.dtype('float64')

# DictionaryType data will be converted to dense numpy array

arr = pa.DictionaryArray.from_arrays(

pa.array([0, 1, 2, 0, 1]), pa.array(['a', 'b', 'c']))

np_arr = np.asarray(arr)

assert np_arr.dtype == np.dtype('object')

assert np_arr.tolist() == ['a', 'b', 'c', 'a', 'b']

@pytest.mark.parametrize('ty', [

None,

pa.null(),

pa.int8(),

pa.string()

])

def test_nulls(ty):

arr = pa.nulls(3, type=ty)

expected = pa.array([None, None, None], type=ty)

assert len(arr) == 3

assert arr.equals(expected)

if ty is None:

assert arr.type == pa.null()

else:

assert arr.type == ty

def test_array_from_scalar():

pytz = pytest.importorskip("pytz")

today = datetime.date.today()

now = datetime.datetime.now()

now_utc = now.replace(tzinfo=pytz.utc)

now_with_tz = now_utc.astimezone(pytz.timezone('US/Eastern'))

oneday = datetime.timedelta(days=1)

cases = [

(None, 1, pa.array([None])),

(None, 10, pa.nulls(10)),

(-1, 3, pa.array([-1, -1, -1], type=pa.int64())),

(2.71, 2, pa.array([2.71, 2.71], type=pa.float64())),

("string", 4, pa.array(["string"] * 4)),

(

pa.scalar(8, type=pa.uint8()),

17,

pa.array([8] * 17, type=pa.uint8())

),

(pa.scalar(None), 3, pa.array([None, None, None])),

(pa.scalar(True), 11, pa.array([True] * 11)),

(today, 2, pa.array([today] * 2)),

(now, 10, pa.array([now] * 10)),

(

now_with_tz,

2,

pa.array(

[now_utc] * 2,

type=pa.timestamp('us', tz=pytz.timezone('US/Eastern'))

)

),

(now.time(), 9, pa.array([now.time()] * 9)),

(oneday, 4, pa.array([oneday] * 4)),

(False, 9, pa.array([False] * 9)),

([1, 2], 2, pa.array([[1, 2], [1, 2]])),

(

pa.scalar([-1, 3], type=pa.large_list(pa.int8())),

5,

pa.array([[-1, 3]] * 5, type=pa.large_list(pa.int8()))

),

({'a': 1, 'b': 2}, 3, pa.array([{'a': 1, 'b': 2}] * 3))

]

for value, size, expected in cases:

arr = pa.repeat(value, size)

assert len(arr) == size

assert arr.type.equals(expected.type)

assert arr.equals(expected)

if expected.type == pa.null():

assert arr.null_count == size

else:

assert arr.null_count == 0

def test_array_from_dictionary_scalar():

dictionary = ['foo', 'bar', 'baz']

arr = pa.DictionaryArray.from_arrays([2, 1, 2, 0], dictionary=dictionary)

result = pa.repeat(arr[0], 5)

expected = pa.DictionaryArray.from_arrays([2] * 5, dictionary=dictionary)

assert result.equals(expected)

result = pa.repeat(arr[3], 5)

expected = pa.DictionaryArray.from_arrays([0] * 5, dictionary=dictionary)

assert result.equals(expected)

def test_array_getitem():

arr = pa.array(range(10, 15))

lst = arr.to_pylist()

for idx in range(-len(arr), len(arr)):

assert arr[idx].as_py() == lst[idx]

for idx in range(-2 * len(arr), -len(arr)):

with pytest.raises(IndexError):

arr[idx]

for idx in range(len(arr), 2 * len(arr)):

with pytest.raises(IndexError):

arr[idx]

# check that numpy scalars are supported

for idx in range(-len(arr), len(arr)):

assert arr[np.int32(idx)].as_py() == lst[idx]

def test_array_slice():

arr = pa.array(range(10))

sliced = arr.slice(2)

expected = pa.array(range(2, 10))

assert sliced.equals(expected)

sliced2 = arr.slice(2, 4)

expected2 = pa.array(range(2, 6))

assert sliced2.equals(expected2)

# 0 offset

assert arr.slice(0).equals(arr)

# Slice past end of array

assert len(arr.slice(len(arr))) == 0

assert len(arr.slice(len(arr) + 2)) == 0

assert len(arr.slice(len(arr) + 2, 100)) == 0

with pytest.raises(IndexError):

arr.slice(-1)

with pytest.raises(ValueError):

arr.slice(2, -1)

# Test slice notation

assert arr[2:].equals(arr.slice(2))

assert arr[2:5].equals(arr.slice(2, 3))

assert arr[-5:].equals(arr.slice(len(arr) - 5))

n = len(arr)

for start in range(-n * 2, n * 2):

for stop in range(-n * 2, n * 2):

res = arr[start:stop]

res.validate()

expected = arr.to_pylist()[start:stop]

assert res.to_pylist() == expected

assert res.to_numpy().tolist() == expected

def test_array_slice_negative_step():

# ARROW-2714

np_arr = np.arange(20)

arr = pa.array(np_arr)

chunked_arr = pa.chunked_array([arr])

cases = [

slice(None, None, -1),

slice(None, 6, -2),

slice(10, 6, -2),

slice(8, None, -2),

slice(2, 10, -2),

slice(10, 2, -2),

slice(None, None, 2),

slice(0, 10, 2),

]

for case in cases:

result = arr[case]

expected = pa.array(np_arr[case])

assert result.equals(expected)

result = pa.record_batch([arr], names=['f0'])[case]

expected = pa.record_batch([expected], names=['f0'])

assert result.equals(expected)

result = chunked_arr[case]

expected = pa.chunked_array([np_arr[case]])

assert result.equals(expected)

def test_array_diff():

# ARROW-6252

arr1 = pa.array(['foo'], type=pa.utf8())

arr2 = pa.array(['foo', 'bar', None], type=pa.utf8())

arr3 = pa.array([1, 2, 3])

arr4 = pa.array([[], [1], None], type=pa.list_(pa.int64()))

assert arr1.diff(arr1) == ''

assert arr1.diff(arr2) == '''

@@ -1, +1 @@

+"bar"

+null

'''

assert arr1.diff(arr3).strip() == '# Array types differed: string vs int64'

assert arr1.diff(arr3).strip() == '# Array types differed: string vs int64'

assert arr1.diff(arr4).strip() == ('# Array types differed: string vs '

'list<item: int64>')

def test_array_iter():

arr = pa.array(range(10))

for i, j in zip(range(10), arr):

assert i == j.as_py()

assert isinstance(arr, Iterable)

def test_struct_array_slice():

# ARROW-2311: slicing nested arrays needs special care

ty = pa.struct([pa.field('a', pa.int8()),

pa.field('b', pa.float32())])

arr = pa.array([(1, 2.5), (3, 4.5), (5, 6.5)], type=ty)

assert arr[1:].to_pylist() == [{'a': 3, 'b': 4.5},

{'a': 5, 'b': 6.5}]

def test_array_factory_invalid_type():

class MyObject:

pass

arr = np.array([MyObject()])

with pytest.raises(ValueError):

pa.array(arr)

def test_array_ref_to_ndarray_base():

arr = np.array([1, 2, 3])

refcount = sys.getrefcount(arr)

arr2 = pa.array(arr) # noqa

assert sys.getrefcount(arr) == (refcount + 1)

def test_array_eq():

# ARROW-2150 / ARROW-9445: we define the __eq__ behavior to be

# data equality (not element-wise equality)

arr1 = pa.array([1, 2, 3], type=pa.int32())

arr2 = pa.array([1, 2, 3], type=pa.int32())

arr3 = pa.array([1, 2, 3], type=pa.int64())

assert (arr1 == arr2) is True

assert (arr1 != arr2) is False

assert (arr1 == arr3) is False

assert (arr1 != arr3) is True

assert (arr1 == 1) is False

assert (arr1 == None) is False # noqa: E711

def test_array_from_buffers():

values_buf = pa.py_buffer(np.int16([4, 5, 6, 7]))

nulls_buf = pa.py_buffer(np.uint8([0b00001101]))

arr = pa.Array.from_buffers(pa.int16(), 4, [nulls_buf, values_buf])

assert arr.type == pa.int16()

assert arr.to_pylist() == [4, None, 6, 7]

arr = pa.Array.from_buffers(pa.int16(), 4, [None, values_buf])

assert arr.type == pa.int16()

assert arr.to_pylist() == [4, 5, 6, 7]

arr = pa.Array.from_buffers(pa.int16(), 3, [nulls_buf, values_buf],

offset=1)

assert arr.type == pa.int16()

assert arr.to_pylist() == [None, 6, 7]

with pytest.raises(TypeError):

pa.Array.from_buffers(pa.int16(), 3, ['', ''], offset=1)

def test_string_binary_from_buffers():

array = pa.array(["a", None, "b", "c"])

buffers = array.buffers()

copied = pa.StringArray.from_buffers(

len(array), buffers[1], buffers[2], buffers[0], array.null_count,

array.offset)

assert copied.to_pylist() == ["a", None, "b", "c"]

binary_copy = pa.Array.from_buffers(pa.binary(), len(array),

array.buffers(), array.null_count,

array.offset)

assert binary_copy.to_pylist() == [b"a", None, b"b", b"c"]

copied = pa.StringArray.from_buffers(

len(array), buffers[1], buffers[2], buffers[0])

assert copied.to_pylist() == ["a", None, "b", "c"]

sliced = array[1:]

buffers = sliced.buffers()

copied = pa.StringArray.from_buffers(

len(sliced), buffers[1], buffers[2], buffers[0], -1, sliced.offset)

assert copied.to_pylist() == [None, "b", "c"]

assert copied.null_count == 1

# Slice but exclude all null entries so that we don't need to pass

# the null bitmap.

sliced = array[2:]

buffers = sliced.buffers()

copied = pa.StringArray.from_buffers(

len(sliced), buffers[1], buffers[2], None, -1, sliced.offset)

assert copied.to_pylist() == ["b", "c"]

assert copied.null_count == 0

@pytest.mark.parametrize('list_type_factory', [pa.list_, pa.large_list])

def test_list_from_buffers(list_type_factory):

ty = list_type_factory(pa.int16())

array = pa.array([[0, 1, 2], None, [], [3, 4, 5]], type=ty)

assert array.type == ty

buffers = array.buffers()

with pytest.raises(ValueError):

# No children

pa.Array.from_buffers(ty, 4, [None, buffers[1]])

child = pa.Array.from_buffers(pa.int16(), 6, buffers[2:])

copied = pa.Array.from_buffers(ty, 4, buffers[:2], children=[child])

assert copied.equals(array)

with pytest.raises(ValueError):

# too many children

pa.Array.from_buffers(ty, 4, [None, buffers[1]],

children=[child, child])

def test_struct_from_buffers():

ty = pa.struct([pa.field('a', pa.int16()), pa.field('b', pa.utf8())])

array = pa.array([{'a': 0, 'b': 'foo'}, None, {'a': 5, 'b': ''}],

type=ty)

buffers = array.buffers()

with pytest.raises(ValueError):

# No children

pa.Array.from_buffers(ty, 3, [None, buffers[1]])

children = [pa.Array.from_buffers(pa.int16(), 3, buffers[1:3]),

pa.Array.from_buffers(pa.utf8(), 3, buffers[3:])]

copied = pa.Array.from_buffers(ty, 3, buffers[:1], children=children)

assert copied.equals(array)

with pytest.raises(ValueError):

# not enough many children

pa.Array.from_buffers(ty, 3, [buffers[0]],

children=children[:1])

def test_struct_from_arrays():

a = pa.array([4, 5, 6], type=pa.int64())

b = pa.array(["bar", None, ""])

c = pa.array([[1, 2], None, [3, None]])

expected_list = [

{'a': 4, 'b': 'bar', 'c': [1, 2]},

{'a': 5, 'b': None, 'c': None},

{'a': 6, 'b': '', 'c': [3, None]},

]

# From field names

arr = pa.StructArray.from_arrays([a, b, c], ["a", "b", "c"])

assert arr.type == pa.struct(

[("a", a.type), ("b", b.type), ("c", c.type)])

assert arr.to_pylist() == expected_list

with pytest.raises(ValueError):

pa.StructArray.from_arrays([a, b, c], ["a", "b"])

arr = pa.StructArray.from_arrays([], [])

assert arr.type == pa.struct([])

assert arr.to_pylist() == []

# From fields

fa = pa.field("a", a.type, nullable=False)

fb = pa.field("b", b.type)

fc = pa.field("c", c.type)

arr = pa.StructArray.from_arrays([a, b, c], fields=[fa, fb, fc])

assert arr.type == pa.struct([fa, fb, fc])

assert not arr.type[0].nullable

assert arr.to_pylist() == expected_list

with pytest.raises(ValueError):

pa.StructArray.from_arrays([a, b, c], fields=[fa, fb])

arr = pa.StructArray.from_arrays([], fields=[])

assert arr.type == pa.struct([])

assert arr.to_pylist() == []

# Inconsistent fields

fa2 = pa.field("a", pa.int32())

with pytest.raises(ValueError, match="int64 vs int32"):

pa.StructArray.from_arrays([a, b, c], fields=[fa2, fb, fc])

arrays = [a, b, c]

fields = [fa, fb, fc]

# With mask

mask = pa.array([True, False, False])

arr = pa.StructArray.from_arrays(arrays, fields=fields, mask=mask)

assert arr.to_pylist() == [None] + expected_list[1:]

arr = pa.StructArray.from_arrays(arrays, names=['a', 'b', 'c'], mask=mask)

assert arr.to_pylist() == [None] + expected_list[1:]

# Bad masks

with pytest.raises(TypeError, match='Mask must be'):

pa.StructArray.from_arrays(arrays, fields, mask=[True, False, False])

with pytest.raises(ValueError, match='not contain nulls'):

pa.StructArray.from_arrays(

arrays, fields, mask=pa.array([True, False, None]))

with pytest.raises(TypeError, match='Mask must be'):

pa.StructArray.from_arrays(

arrays, fields, mask=pa.chunked_array([mask]))

# Non-empty array with no fields https://github.com/apache/arrow/issues/15109

arr = pa.StructArray.from_arrays([], [], mask=mask)

assert arr.is_null() == mask

assert arr.to_pylist() == [None, {}, {}]

def test_struct_array_from_chunked():

# ARROW-11780

# Check that we don't segfault when trying to build

# a StructArray from a chunked array.

chunked_arr = pa.chunked_array([[1, 2, 3], [4, 5, 6]])

with pytest.raises(TypeError, match="Expected Array"):

pa.StructArray.from_arrays([chunked_arr], ["foo"])

@pytest.mark.parametrize("offset", (0, 1))

def test_dictionary_from_buffers(offset):

a = pa.array(["one", "two", "three", "two", "one"]).dictionary_encode()

b = pa.DictionaryArray.from_buffers(a.type, len(a)-offset,

a.indices.buffers(), a.dictionary,

offset=offset)

assert a[offset:] == b

def test_dictionary_from_numpy():

indices = np.repeat([0, 1, 2], 2)

dictionary = np.array(['foo', 'bar', 'baz'], dtype=object)

mask = np.array([False, False, True, False, False, False])

d1 = pa.DictionaryArray.from_arrays(indices, dictionary)

d2 = pa.DictionaryArray.from_arrays(indices, dictionary, mask=mask)

assert d1.indices.to_pylist() == indices.tolist()

assert d1.indices.to_pylist() == indices.tolist()

assert d1.dictionary.to_pylist() == dictionary.tolist()

assert d2.dictionary.to_pylist() == dictionary.tolist()

for i in range(len(indices)):

assert d1[i].as_py() == dictionary[indices[i]]

if mask[i]:

assert d2[i].as_py() is None

else:

assert d2[i].as_py() == dictionary[indices[i]]

def test_dictionary_to_numpy():

expected = pa.array(

["foo", "bar", None, "foo"]

).to_numpy(zero_copy_only=False)

a = pa.DictionaryArray.from_arrays(

pa.array([0, 1, None, 0]),

pa.array(['foo', 'bar'])

)

np.testing.assert_array_equal(a.to_numpy(zero_copy_only=False),

expected)

with pytest.raises(pa.ArrowInvalid):

# If this would be changed to no longer raise in the future,

# ensure to test the actual result because, currently, to_numpy takes

# for granted that when zero_copy_only=True there will be no nulls

# (it's the decoding of the DictionaryArray that handles the nulls and

# this is only activated with zero_copy_only=False)

a.to_numpy(zero_copy_only=True)

anonulls = pa.DictionaryArray.from_arrays(

pa.array([0, 1, 1, 0]),

pa.array(['foo', 'bar'])

)

expected = pa.array(

["foo", "bar", "bar", "foo"]

).to_numpy(zero_copy_only=False)

np.testing.assert_array_equal(anonulls.to_numpy(zero_copy_only=False),

expected)

with pytest.raises(pa.ArrowInvalid):

anonulls.to_numpy(zero_copy_only=True)

afloat = pa.DictionaryArray.from_arrays(

pa.array([0, 1, 1, 0]),

pa.array([13.7, 11.0])

)

expected = pa.array([13.7, 11.0, 11.0, 13.7]).to_numpy()

np.testing.assert_array_equal(afloat.to_numpy(zero_copy_only=True),

expected)

np.testing.assert_array_equal(afloat.to_numpy(zero_copy_only=False),

expected)

afloat2 = pa.DictionaryArray.from_arrays(

pa.array([0, 1, None, 0]),

pa.array([13.7, 11.0])

)

expected = pa.array(

[13.7, 11.0, None, 13.7]

).to_numpy(zero_copy_only=False)

np.testing.assert_allclose(

afloat2.to_numpy(zero_copy_only=False),

expected,

equal_nan=True

)

# Testing for integers can reveal problems related to dealing

# with None values, as a numpy array of int dtype

# can't contain NaN nor None.

aints = pa.DictionaryArray.from_arrays(

pa.array([0, 1, None, 0]),

pa.array([7, 11])

)

expected = pa.array([7, 11, None, 7]).to_numpy(zero_copy_only=False)

np.testing.assert_allclose(

aints.to_numpy(zero_copy_only=False),

expected,

equal_nan=True

)

def test_dictionary_from_boxed_arrays():

indices = np.repeat([0, 1, 2], 2)

dictionary = np.array(['foo', 'bar', 'baz'], dtype=object)

iarr = pa.array(indices)

darr = pa.array(dictionary)

d1 = pa.DictionaryArray.from_arrays(iarr, darr)

assert d1.indices.to_pylist() == indices.tolist()

assert d1.dictionary.to_pylist() == dictionary.tolist()

for i in range(len(indices)):

assert d1[i].as_py() == dictionary[indices[i]]

def test_dictionary_from_arrays_boundscheck():

indices1 = pa.array([0, 1, 2, 0, 1, 2])

indices2 = pa.array([0, -1, 2])

indices3 = pa.array([0, 1, 2, 3])

dictionary = pa.array(['foo', 'bar', 'baz'])

# Works fine

pa.DictionaryArray.from_arrays(indices1, dictionary)

with pytest.raises(pa.ArrowException):

pa.DictionaryArray.from_arrays(indices2, dictionary)

with pytest.raises(pa.ArrowException):

pa.DictionaryArray.from_arrays(indices3, dictionary)

# If we are confident that the indices are "safe" we can pass safe=False to

# disable the boundschecking

pa.DictionaryArray.from_arrays(indices2, dictionary, safe=False)

def test_dictionary_indices():

# https://issues.apache.org/jira/browse/ARROW-6882

indices = pa.array([0, 1, 2, 0, 1, 2])

dictionary = pa.array(['foo', 'bar', 'baz'])

arr = pa.DictionaryArray.from_arrays(indices, dictionary)

arr.indices.validate(full=True)

@pytest.mark.parametrize(('list_array_type', 'list_type_factory'),

[(pa.ListArray, pa.list_),

(pa.LargeListArray, pa.large_list)])

def test_list_from_arrays(list_array_type, list_type_factory):

offsets_arr = np.array([0, 2, 5, 8], dtype='i4')

offsets = pa.array(offsets_arr, type='int32')

pyvalues = [b'a', b'b', b'c', b'd', b'e', b'f', b'g', b'h']

values = pa.array(pyvalues, type='binary')

result = list_array_type.from_arrays(offsets, values)

expected = pa.array([pyvalues[:2], pyvalues[2:5], pyvalues[5:8]],

type=list_type_factory(pa.binary()))

assert result.equals(expected)

# With specified type

typ = list_type_factory(pa.field("name", pa.binary()))

result = list_array_type.from_arrays(offsets, values, typ)

assert result.type == typ

assert result.type.value_field.name == "name"

# With nulls

offsets = [0, None, 2, 6]

values = [b'a', b'b', b'c', b'd', b'e', b'f']

result = list_array_type.from_arrays(offsets, values)

expected = pa.array([values[:2], None, values[2:]],

type=list_type_factory(pa.binary()))

assert result.equals(expected)

# Another edge case

offsets2 = [0, 2, None, 6]

result = list_array_type.from_arrays(offsets2, values)

expected = pa.array([values[:2], values[2:], None],

type=list_type_factory(pa.binary()))

assert result.equals(expected)

# raise on invalid array

offsets = [1, 3, 10]

values = np.arange(5)

with pytest.raises(ValueError):

list_array_type.from_arrays(offsets, values)

# Non-monotonic offsets

offsets = [0, 3, 2, 6]

values = list(range(6))

result = list_array_type.from_arrays(offsets, values)

with pytest.raises(ValueError):

result.validate(full=True)

# mismatching type

typ = list_type_factory(pa.binary())

with pytest.raises(TypeError):

list_array_type.from_arrays(offsets, values, type=typ)

@pytest.mark.parametrize(('list_array_type', 'list_type_factory'), (

(pa.ListArray, pa.list_),

(pa.LargeListArray, pa.large_list)

))

@pytest.mark.parametrize("arr", (

[None, [0]],

[None, [0, None], [0]],

[[0], [1]],

))

def test_list_array_types_from_arrays(

list_array_type, list_type_factory, arr

):

arr = pa.array(arr, list_type_factory(pa.int8()))

reconstructed_arr = list_array_type.from_arrays(

arr.offsets, arr.values, mask=arr.is_null())

assert arr == reconstructed_arr

@pytest.mark.parametrize(('list_array_type', 'list_type_factory'), (

(pa.ListArray, pa.list_),

(pa.LargeListArray, pa.large_list)

))

def test_list_array_types_from_arrays_fail(list_array_type, list_type_factory):

# Fail when manual offsets include nulls and mask passed

# ListArray.offsets doesn't report nulls.

# This test case arr.offsets == [0, 1, 1, 3, 4]

arr = pa.array([[0], None, [0, None], [0]], list_type_factory(pa.int8()))

offsets = pa.array([0, None, 1, 3, 4])

# Using array's offset has no nulls; gives empty lists on top level

reconstructed_arr = list_array_type.from_arrays(arr.offsets, arr.values)

assert reconstructed_arr.to_pylist() == [[0], [], [0, None], [0]]

# Manually specifiying offsets (with nulls) is same as mask at top level

reconstructed_arr = list_array_type.from_arrays(offsets, arr.values)

assert arr == reconstructed_arr

reconstructed_arr = list_array_type.from_arrays(arr.offsets,

arr.values,

mask=arr.is_null())

assert arr == reconstructed_arr

# But using both is ambiguous, in this case `offsets` has nulls

with pytest.raises(ValueError, match="Ambiguous to specify both "):

list_array_type.from_arrays(offsets, arr.values, mask=arr.is_null())

# Not supported to reconstruct from a slice.

arr_slice = arr[1:]

msg = "Null bitmap with offsets slice not supported."

with pytest.raises(NotImplementedError, match=msg):

list_array_type.from_arrays(

arr_slice.offsets, arr_slice.values, mask=arr_slice.is_null())

def test_map_labelled():

# ARROW-13735

t = pa.map_(pa.field("name", "string", nullable=False), "int64")

arr = pa.array([[('a', 1), ('b', 2)], [('c', 3)]], type=t)

assert arr.type.key_field == pa.field("name", pa.utf8(), nullable=False)

assert arr.type.item_field == pa.field("value", pa.int64())

assert len(arr) == 2

def test_map_from_dict():

# ARROW-17832