Skip to content

Commit 9752c34

Browse files
authored
GH-35289: [Python] Support large variable width types in numpy conversion (#36701)
### Rationale for this change Add support for LargeBinaryType, BinaryViewType, LargeStringType, and StringViewType in NumPy to Arrow conversion. ### What changes are included in this PR? Adds new `Visit` methods in `NumPyConverter` for `LargeStringType`, `StringViewType`, `LargeBinaryType`, and `BinaryViewType`. These are built on top of new templated `VisitString` and `VisitBinary` methods. Also adds the Arrow -> NumPy type map for the large binary types. ### Are these changes tested? New test added showing a string and binary array over 2 GiB is valid and still a single chunk. Also added the new Arrow to NumPy maps to a schema test. ### Are there any user-facing changes? Adds support for converting NumPy string/binary lists to large binary types. * Closes: #35289 * GitHub Issue: #35289 Authored-by: Adam Binford <[email protected]> Signed-off-by: AlenkaF <[email protected]>
1 parent 30adc91 commit 9752c34

4 files changed

Lines changed: 155 additions & 50 deletions

File tree

‎python/pyarrow/src/arrow/python/numpy_to_arrow.cc‎

Lines changed: 86 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -225,9 +225,13 @@ class NumPyConverter {
225225

226226
// NumPy ascii string arrays
227227
Status Visit(const BinaryType& type);
228+
Status Visit(const LargeBinaryType& type);
229+
Status Visit(const BinaryViewType& type);
228230

229231
// NumPy unicode arrays
230232
Status Visit(const StringType& type);
233+
Status Visit(const LargeStringType& type);
234+
Status Visit(const StringViewType& type);
231235

232236
Status Visit(const StructType& type);
233237

@@ -285,6 +289,12 @@ class NumPyConverter {
285289
return PushArray(arr_data);
286290
}
287291

292+
template <typename T>
293+
Status VisitBinary(T* builder);
294+
295+
template <typename T>
296+
Status VisitString(T* builder);
297+
288298
Status TypeNotImplemented(std::string type_name) {
289299
return Status::NotImplemented("NumPyConverter doesn't implement <", type_name,
290300
"> conversion. ");
@@ -554,24 +564,23 @@ inline Status NumPyConverter::ConvertData<Date64Type>(std::shared_ptr<Buffer>* d
554564
// Create 16MB chunks for binary data
555565
constexpr int32_t kBinaryChunksize = 1 << 24;
556566

557-
Status NumPyConverter::Visit(const BinaryType& type) {
558-
::arrow::internal::ChunkedBinaryBuilder builder(kBinaryChunksize, pool_);
559-
567+
template <typename T>
568+
Status NumPyConverter::VisitBinary(T* builder) {
560569
auto data = reinterpret_cast<const uint8_t*>(PyArray_DATA(arr_));
561570

562-
auto AppendNotNull = [&builder, this](const uint8_t* data) {
571+
auto AppendNotNull = [builder, this](const uint8_t* data) {
563572
// This is annoying. NumPy allows strings to have nul-terminators, so
564573
// we must check for them here
565574
const size_t item_size =
566575
strnlen(reinterpret_cast<const char*>(data), static_cast<size_t>(itemsize_));
567-
return builder.Append(data, static_cast<int32_t>(item_size));
576+
return builder->Append(data, static_cast<int32_t>(item_size));
568577
};
569578

570579
if (mask_ != nullptr) {
571580
Ndarray1DIndexer<uint8_t> mask_values(mask_);
572581
for (int64_t i = 0; i < length_; ++i) {
573582
if (mask_values[i]) {
574-
RETURN_NOT_OK(builder.AppendNull());
583+
RETURN_NOT_OK(builder->AppendNull());
575584
} else {
576585
RETURN_NOT_OK(AppendNotNull(data));
577586
}
@@ -584,6 +593,14 @@ Status NumPyConverter::Visit(const BinaryType& type) {
584593
}
585594
}
586595

596+
return Status::OK();
597+
}
598+
599+
Status NumPyConverter::Visit(const BinaryType& type) {
600+
::arrow::internal::ChunkedBinaryBuilder builder(kBinaryChunksize, pool_);
601+
602+
RETURN_NOT_OK(VisitBinary(&builder));
603+
587604
ArrayVector result;
588605
RETURN_NOT_OK(builder.Finish(&result));
589606
for (auto arr : result) {
@@ -592,6 +609,26 @@ Status NumPyConverter::Visit(const BinaryType& type) {
592609
return Status::OK();
593610
}
594611

612+
Status NumPyConverter::Visit(const LargeBinaryType& type) {
613+
::arrow::LargeBinaryBuilder builder(pool_);
614+
615+
RETURN_NOT_OK(VisitBinary(&builder));
616+
617+
std::shared_ptr<Array> result;
618+
RETURN_NOT_OK(builder.Finish(&result));
619+
return PushArray(result->data());
620+
}
621+
622+
Status NumPyConverter::Visit(const BinaryViewType& type) {
623+
::arrow::BinaryViewBuilder builder(pool_);
624+
625+
RETURN_NOT_OK(VisitBinary(&builder));
626+
627+
std::shared_ptr<Array> result;
628+
RETURN_NOT_OK(builder.Finish(&result));
629+
return PushArray(result->data());
630+
}
631+
595632
Status NumPyConverter::Visit(const FixedSizeBinaryType& type) {
596633
auto byte_width = type.byte_width();
597634

@@ -631,8 +668,8 @@ namespace {
631668
// NumPy unicode is UCS4/UTF32 always
632669
constexpr int kNumPyUnicodeSize = 4;
633670

634-
Status AppendUTF32(const char* data, int64_t itemsize, int byteorder,
635-
::arrow::internal::ChunkedStringBuilder* builder) {
671+
template <typename T>
672+
Status AppendUTF32(const char* data, int64_t itemsize, int byteorder, T* builder) {
636673
// The binary \x00\x00\x00\x00 indicates a nul terminator in NumPy unicode,
637674
// so we need to detect that here to truncate if necessary. Yep.
638675
Py_ssize_t actual_length = 0;
@@ -660,11 +697,8 @@ Status AppendUTF32(const char* data, int64_t itemsize, int byteorder,
660697

661698
} // namespace
662699

663-
Status NumPyConverter::Visit(const StringType& type) {
664-
util::InitializeUTF8();
665-
666-
::arrow::internal::ChunkedStringBuilder builder(kBinaryChunksize, pool_);
667-
700+
template <typename T>
701+
Status NumPyConverter::VisitString(T* builder) {
668702
auto data = reinterpret_cast<const uint8_t*>(PyArray_DATA(arr_));
669703

670704
char numpy_byteorder = dtype_->byteorder;
@@ -708,23 +742,23 @@ Status NumPyConverter::Visit(const StringType& type) {
708742
auto AppendNonNullValue = [&](const uint8_t* data) {
709743
if (is_binary_type) {
710744
if (ARROW_PREDICT_TRUE(util::ValidateUTF8(data, itemsize_))) {
711-
return builder.Append(data, static_cast<int32_t>(itemsize_));
745+
return builder->Append(data, static_cast<int32_t>(itemsize_));
712746
} else {
713747
return Status::Invalid("Encountered non-UTF8 binary value: ",
714748
HexEncode(data, itemsize_));
715749
}
716750
} else {
717751
// is_unicode_type case
718752
return AppendUTF32(reinterpret_cast<const char*>(data), itemsize_, byteorder,
719-
&builder);
753+
builder);
720754
}
721755
};
722756

723757
if (mask_ != nullptr) {
724758
Ndarray1DIndexer<uint8_t> mask_values(mask_);
725759
for (int64_t i = 0; i < length_; ++i) {
726760
if (mask_values[i]) {
727-
RETURN_NOT_OK(builder.AppendNull());
761+
RETURN_NOT_OK(builder->AppendNull());
728762
} else {
729763
RETURN_NOT_OK(AppendNonNullValue(data));
730764
}
@@ -737,6 +771,16 @@ Status NumPyConverter::Visit(const StringType& type) {
737771
}
738772
}
739773

774+
return Status::OK();
775+
}
776+
777+
Status NumPyConverter::Visit(const StringType& type) {
778+
util::InitializeUTF8();
779+
780+
::arrow::internal::ChunkedStringBuilder builder(kBinaryChunksize, pool_);
781+
782+
RETURN_NOT_OK(VisitString(&builder));
783+
740784
ArrayVector result;
741785
RETURN_NOT_OK(builder.Finish(&result));
742786
for (auto arr : result) {
@@ -745,6 +789,32 @@ Status NumPyConverter::Visit(const StringType& type) {
745789
return Status::OK();
746790
}
747791

792+
Status NumPyConverter::Visit(const LargeStringType& type) {
793+
util::InitializeUTF8();
794+
795+
::arrow::LargeStringBuilder builder(pool_);
796+
797+
RETURN_NOT_OK(VisitString(&builder));
798+
799+
std::shared_ptr<Array> result;
800+
RETURN_NOT_OK(builder.Finish(&result));
801+
RETURN_NOT_OK(PushArray(result->data()));
802+
return Status::OK();
803+
}
804+
805+
Status NumPyConverter::Visit(const StringViewType& type) {
806+
util::InitializeUTF8();
807+
808+
::arrow::StringViewBuilder builder(pool_);
809+
810+
RETURN_NOT_OK(VisitString(&builder));
811+
812+
std::shared_ptr<Array> result;
813+
RETURN_NOT_OK(builder.Finish(&result));
814+
RETURN_NOT_OK(PushArray(result->data()));
815+
return Status::OK();
816+
}
817+
748818
Status NumPyConverter::Visit(const StructType& type) {
749819
std::vector<NumPyConverter> sub_converters;
750820
std::vector<OwnedRefNoGIL> sub_arrays;

‎python/pyarrow/tests/test_array.py‎

Lines changed: 61 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -2498,32 +2498,40 @@ def test_array_from_numpy_timedelta_incorrect_unit():
24982498

24992499

25002500
@pytest.mark.numpy
2501-
def test_array_from_numpy_ascii():
2501+
@pytest.mark.parametrize('binary_type', [
2502+
None,
2503+
pa.binary(),
2504+
pa.large_binary(),
2505+
pa.binary_view()])
2506+
def test_array_from_numpy_ascii(binary_type):
2507+
# Default when no type is specified should be binary
2508+
expected_type = binary_type or pa.binary()
2509+
25022510
arr = np.array(['abcde', 'abc', ''], dtype='|S5')
25032511

2504-
arrow_arr = pa.array(arr)
2505-
assert arrow_arr.type == 'binary'
2506-
expected = pa.array(['abcde', 'abc', ''], type='binary')
2512+
arrow_arr = pa.array(arr, binary_type)
2513+
assert arrow_arr.type == expected_type
2514+
expected = pa.array(['abcde', 'abc', ''], type=expected_type)
25072515
assert arrow_arr.equals(expected)
25082516

25092517
mask = np.array([False, True, False])
2510-
arrow_arr = pa.array(arr, mask=mask)
2511-
expected = pa.array(['abcde', None, ''], type='binary')
2518+
arrow_arr = pa.array(arr, binary_type, mask=mask)
2519+
expected = pa.array(['abcde', None, ''], type=expected_type)
25122520
assert arrow_arr.equals(expected)
25132521

25142522
# Strided variant
25152523
arr = np.array(['abcde', 'abc', ''] * 5, dtype='|S5')[::2]
25162524
mask = np.array([False, True, False] * 5)[::2]
2517-
arrow_arr = pa.array(arr, mask=mask)
2525+
arrow_arr = pa.array(arr, binary_type, mask=mask)
25182526

25192527
expected = pa.array(['abcde', '', None, 'abcde', '', None, 'abcde', ''],
2520-
type='binary')
2528+
type=expected_type)
25212529
assert arrow_arr.equals(expected)
25222530

25232531
# 0 itemsize
25242532
arr = np.array(['', '', ''], dtype='|S0')
2525-
arrow_arr = pa.array(arr)
2526-
expected = pa.array(['', '', ''], type='binary')
2533+
arrow_arr = pa.array(arr, binary_type)
2534+
expected = pa.array(['', '', ''], type=expected_type)
25272535
assert arrow_arr.equals(expected)
25282536

25292537

@@ -2643,35 +2651,43 @@ def test_interval_array_from_dateoffset():
26432651

26442652

26452653
@pytest.mark.numpy
2646-
def test_array_from_numpy_unicode():
2654+
@pytest.mark.parametrize('string_type', [
2655+
None,
2656+
pa.utf8(),
2657+
pa.large_utf8(),
2658+
pa.string_view()])
2659+
def test_array_from_numpy_unicode(string_type):
2660+
# Default when no type is specified should be utf8
2661+
expected_type = string_type or pa.utf8()
2662+
26472663
dtypes = ['<U5', '>U5']
26482664

26492665
for dtype in dtypes:
26502666
arr = np.array(['abcde', 'abc', ''], dtype=dtype)
26512667

2652-
arrow_arr = pa.array(arr)
2653-
assert arrow_arr.type == 'utf8'
2654-
expected = pa.array(['abcde', 'abc', ''], type='utf8')
2668+
arrow_arr = pa.array(arr, string_type)
2669+
assert arrow_arr.type == expected_type
2670+
expected = pa.array(['abcde', 'abc', ''], type=expected_type)
26552671
assert arrow_arr.equals(expected)
26562672

26572673
mask = np.array([False, True, False])
2658-
arrow_arr = pa.array(arr, mask=mask)
2659-
expected = pa.array(['abcde', None, ''], type='utf8')
2674+
arrow_arr = pa.array(arr, string_type, mask=mask)
2675+
expected = pa.array(['abcde', None, ''], type=expected_type)
26602676
assert arrow_arr.equals(expected)
26612677

26622678
# Strided variant
26632679
arr = np.array(['abcde', 'abc', ''] * 5, dtype=dtype)[::2]
26642680
mask = np.array([False, True, False] * 5)[::2]
2665-
arrow_arr = pa.array(arr, mask=mask)
2681+
arrow_arr = pa.array(arr, string_type, mask=mask)
26662682

26672683
expected = pa.array(['abcde', '', None, 'abcde', '', None,
2668-
'abcde', ''], type='utf8')
2684+
'abcde', ''], type=expected_type)
26692685
assert arrow_arr.equals(expected)
26702686

26712687
# 0 itemsize
26722688
arr = np.array(['', '', ''], dtype='<U0')
2673-
arrow_arr = pa.array(arr)
2674-
expected = pa.array(['', '', ''], type='utf8')
2689+
arrow_arr = pa.array(arr, string_type)
2690+
expected = pa.array(['', '', ''], type=expected_type)
26752691
assert arrow_arr.equals(expected)
26762692

26772693

@@ -3268,8 +3284,9 @@ def test_array_from_numpy_str_utf8():
32683284
@pytest.mark.numpy
32693285
@pytest.mark.slow
32703286
@pytest.mark.large_memory
3271-
def test_numpy_binary_overflow_to_chunked():
3272-
# ARROW-3762, ARROW-5966
3287+
@pytest.mark.parametrize('large_types', [False, True])
3288+
def test_numpy_binary_overflow_to_chunked(large_types):
3289+
# ARROW-3762, ARROW-5966, GH-35289
32733290

32743291
# 2^31 + 1 bytes
32753292
values = [b'x']
@@ -3286,24 +3303,34 @@ def test_numpy_binary_overflow_to_chunked():
32863303
unicode_values += [unicode_unique_strings[i % 10]
32873304
for i in range(1 << 11)]
32883305

3289-
for case, ex_type in [(values, pa.binary()),
3290-
(unicode_values, pa.utf8())]:
3306+
binary_type = pa.large_binary() if large_types else pa.binary()
3307+
string_type = pa.large_utf8() if large_types else pa.utf8()
3308+
for case, ex_type in [(values, binary_type),
3309+
(unicode_values, string_type)]:
32913310
arr = np.array(case)
3292-
arrow_arr = pa.array(arr)
3311+
arrow_arr = pa.array(arr, ex_type)
32933312
arr = None
32943313

3295-
assert isinstance(arrow_arr, pa.ChunkedArray)
32963314
assert arrow_arr.type == ex_type
3315+
if large_types:
3316+
# Large types shouldn't be chunked
3317+
assert isinstance(arrow_arr, pa.Array)
3318+
3319+
for i in range(len(arrow_arr)):
3320+
val = arrow_arr[i]
3321+
assert val.as_py() == case[i]
3322+
else:
3323+
assert isinstance(arrow_arr, pa.ChunkedArray)
32973324

3298-
# Split up into 16MB chunks. 128 * 16 = 2048, so 129
3299-
assert arrow_arr.num_chunks == 129
3325+
# Split up into 16MB chunks. 128 * 16 = 2048, so 129
3326+
assert arrow_arr.num_chunks == 129
33003327

3301-
value_index = 0
3302-
for i in range(arrow_arr.num_chunks):
3303-
chunk = arrow_arr.chunk(i)
3304-
for val in chunk:
3305-
assert val.as_py() == case[value_index]
3306-
value_index += 1
3328+
value_index = 0
3329+
for i in range(arrow_arr.num_chunks):
3330+
chunk = arrow_arr.chunk(i)
3331+
for val in chunk:
3332+
assert val.as_py() == case[value_index]
3333+
value_index += 1
33073334

33083335

33093336
@pytest.mark.large_memory

‎python/pyarrow/tests/test_schema.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,8 +75,12 @@ def test_type_to_pandas_dtype():
7575
(pa.date64(), M8),
7676
(pa.timestamp('ms'), M8),
7777
(pa.binary(), np.object_),
78+
(pa.large_binary(), np.object_),
79+
(pa.binary_view(), np.object_),
7880
(pa.binary(12), np.object_),
7981
(pa.string(), np.object_),
82+
(pa.large_string(), np.object_),
83+
(pa.string_view(), np.object_),
8084
(pa.list_(pa.int8()), np.object_),
8185
# (pa.list_(pa.int8(), 2), np.object_), # TODO needs pandas conversion
8286
(pa.map_(pa.int64(), pa.float64()), np.object_),

‎python/pyarrow/types.pxi‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,8 +69,12 @@ def _get_pandas_type_map():
6969
'ns': np.dtype('timedelta64[ns]'),
7070
},
7171
_Type_BINARY: np.object_,
72+
_Type_LARGE_BINARY: np.object_,
73+
_Type_BINARY_VIEW: np.object_,
7274
_Type_FIXED_SIZE_BINARY: np.object_,
7375
_Type_STRING: np.object_,
76+
_Type_LARGE_STRING: np.object_,
77+
_Type_STRING_VIEW: np.object_,
7478
_Type_LIST: np.object_,
7579
_Type_MAP: np.object_,
7680
_Type_DECIMAL32: np.object_,

0 commit comments

Comments
 (0)