Skip to content

Commit b116b8a

Browse files
GH-21761: [Python] accept pyarrow scalars in array constructor (#36162)
### Rationale for this change Currently, `pyarrow.array `doesn't accept list of pyarrow Scalars and this PR adds a check to allow that. * Closes: #21761 Lead-authored-by: AlenkaF <[email protected]> Co-authored-by: Alenka Frim <[email protected]> Co-authored-by: Joris Van den Bossche <[email protected]> Signed-off-by: Joris Van den Bossche <[email protected]>
1 parent 947a446 commit b116b8a

3 files changed

Lines changed: 202 additions & 0 deletions

File tree

‎python/pyarrow/src/arrow/python/inference.cc‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
#include <utility>
2828
#include <vector>
2929

30+
#include "arrow/scalar.h"
3031
#include "arrow/status.h"
3132
#include "arrow/util/decimal.h"
3233
#include "arrow/util/logging.h"
@@ -340,6 +341,7 @@ class TypeInferrer {
340341
decimal_count_(0),
341342
list_count_(0),
342343
struct_count_(0),
344+
arrow_scalar_count_(0),
343345
numpy_dtype_count_(0),
344346
interval_count_(0),
345347
max_decimal_metadata_(std::numeric_limits<int32_t>::min(),
@@ -391,6 +393,8 @@ class TypeInferrer {
391393
} else if (PyUnicode_Check(obj)) {
392394
++unicode_count_;
393395
*keep_going = make_unions_;
396+
} else if (arrow::py::is_scalar(obj)) {
397+
RETURN_NOT_OK(VisitArrowScalar(obj, keep_going));
394398
} else if (PyArray_CheckAnyScalarExact(obj)) {
395399
RETURN_NOT_OK(VisitDType(PyArray_DescrFromScalar(obj), keep_going));
396400
} else if (PySet_Check(obj) || (Py_TYPE(obj) == &PyDictValues_Type)) {
@@ -455,6 +459,12 @@ class TypeInferrer {
455459

456460
RETURN_NOT_OK(Validate());
457461

462+
if (arrow_scalar_count_ > 0 && arrow_scalar_count_ + none_count_ != total_count_) {
463+
return Status::Invalid(
464+
"pyarrow scalars cannot be mixed "
465+
"with other Python scalar values currently");
466+
}
467+
458468
if (numpy_dtype_count_ > 0) {
459469
// All NumPy scalars and Nones/nulls
460470
if (numpy_dtype_count_ + none_count_ == total_count_) {
@@ -534,6 +544,8 @@ class TypeInferrer {
534544
*out = utf8();
535545
} else if (interval_count_) {
536546
*out = month_day_nano_interval();
547+
} else if (arrow_scalar_count_) {
548+
*out = scalar_type_;
537549
} else {
538550
*out = null();
539551
}
@@ -560,6 +572,17 @@ class TypeInferrer {
560572
return Status::OK();
561573
}
562574

575+
Status VisitArrowScalar(PyObject* obj, bool* keep_going /* unused */) {
576+
ARROW_ASSIGN_OR_RAISE(auto scalar, arrow::py::unwrap_scalar(obj));
577+
// Check that all the scalar types for the sequence are the same
578+
if (arrow_scalar_count_ > 0 && *scalar->type != *scalar_type_) {
579+
return internal::InvalidValue(obj, "cannot mix scalars with different types");
580+
}
581+
scalar_type_ = scalar->type;
582+
++arrow_scalar_count_;
583+
return Status::OK();
584+
}
585+
563586
Status VisitDType(PyArray_Descr* dtype, bool* keep_going) {
564587
// Continue visiting dtypes for now.
565588
// TODO(wesm): devise approach for unions
@@ -675,10 +698,12 @@ class TypeInferrer {
675698
int64_t decimal_count_;
676699
int64_t list_count_;
677700
int64_t struct_count_;
701+
int64_t arrow_scalar_count_;
678702
int64_t numpy_dtype_count_;
679703
int64_t interval_count_;
680704
std::unique_ptr<TypeInferrer> list_inferrer_;
681705
std::map<std::string, TypeInferrer> struct_inferrers_;
706+
std::shared_ptr<DataType> scalar_type_;
682707

683708
// If we observe a strongly-typed value in e.g. a NumPy array, we can store
684709
// it here to skip the type counting logic above

‎python/pyarrow/src/arrow/python/python_to_arrow.cc‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
#include <vector>
2929

3030
#include "arrow/array.h"
31+
#include "arrow/array/builder_base.h"
3132
#include "arrow/array/builder_binary.h"
3233
#include "arrow/array/builder_decimal.h"
3334
#include "arrow/array/builder_dict.h"
@@ -36,6 +37,7 @@
3637
#include "arrow/array/builder_time.h"
3738
#include "arrow/chunked_array.h"
3839
#include "arrow/result.h"
40+
#include "arrow/scalar.h"
3941
#include "arrow/status.h"
4042
#include "arrow/type.h"
4143
#include "arrow/type_traits.h"
@@ -599,6 +601,15 @@ class PyPrimitiveConverter<T, enable_if_null<T>>
599601
Status Append(PyObject* value) override {
600602
if (PyValue::IsNull(this->options_, value)) {
601603
return this->primitive_builder_->AppendNull();
604+
} else if (arrow::py::is_scalar(value)) {
605+
ARROW_ASSIGN_OR_RAISE(std::shared_ptr<Scalar> scalar,
606+
arrow::py::unwrap_scalar(value));
607+
if (scalar->is_valid) {
608+
return Status::Invalid("Cannot append scalar of type ", scalar->type->ToString(),
609+
" to builder for type null");
610+
} else {
611+
return this->primitive_builder_->AppendNull();
612+
}
602613
} else {
603614
ARROW_ASSIGN_OR_RAISE(
604615
auto converted, PyValue::Convert(this->primitive_type_, this->options_, value));
@@ -620,6 +631,10 @@ class PyPrimitiveConverter<
620631
// rely on the Unsafe builder API which improves the performance.
621632
if (PyValue::IsNull(this->options_, value)) {
622633
this->primitive_builder_->UnsafeAppendNull();
634+
} else if (arrow::py::is_scalar(value)) {
635+
ARROW_ASSIGN_OR_RAISE(std::shared_ptr<Scalar> scalar,
636+
arrow::py::unwrap_scalar(value));
637+
ARROW_RETURN_NOT_OK(this->primitive_builder_->AppendScalar(*scalar));
623638
} else {
624639
ARROW_ASSIGN_OR_RAISE(
625640
auto converted, PyValue::Convert(this->primitive_type_, this->options_, value));
@@ -637,6 +652,10 @@ class PyPrimitiveConverter<
637652
Status Append(PyObject* value) override {
638653
if (PyValue::IsNull(this->options_, value)) {
639654
this->primitive_builder_->UnsafeAppendNull();
655+
} else if (arrow::py::is_scalar(value)) {
656+
ARROW_ASSIGN_OR_RAISE(std::shared_ptr<Scalar> scalar,
657+
arrow::py::unwrap_scalar(value));
658+
ARROW_RETURN_NOT_OK(this->primitive_builder_->AppendScalar(*scalar));
640659
} else {
641660
ARROW_ASSIGN_OR_RAISE(
642661
auto converted, PyValue::Convert(this->primitive_type_, this->options_, value));
@@ -659,6 +678,10 @@ class PyPrimitiveConverter<T, enable_if_t<std::is_same<T, FixedSizeBinaryType>::
659678
Status Append(PyObject* value) override {
660679
if (PyValue::IsNull(this->options_, value)) {
661680
this->primitive_builder_->UnsafeAppendNull();
681+
} else if (arrow::py::is_scalar(value)) {
682+
ARROW_ASSIGN_OR_RAISE(std::shared_ptr<Scalar> scalar,
683+
arrow::py::unwrap_scalar(value));
684+
ARROW_RETURN_NOT_OK(this->primitive_builder_->AppendScalar(*scalar));
662685
} else {
663686
ARROW_RETURN_NOT_OK(
664687
PyValue::Convert(this->primitive_type_, this->options_, value, view_));
@@ -681,6 +704,10 @@ class PyPrimitiveConverter<T, enable_if_base_binary<T>>
681704
Status Append(PyObject* value) override {
682705
if (PyValue::IsNull(this->options_, value)) {
683706
this->primitive_builder_->UnsafeAppendNull();
707+
} else if (arrow::py::is_scalar(value)) {
708+
ARROW_ASSIGN_OR_RAISE(std::shared_ptr<Scalar> scalar,
709+
arrow::py::unwrap_scalar(value));
710+
ARROW_RETURN_NOT_OK(this->primitive_builder_->AppendScalar(*scalar));
684711
} else {
685712
ARROW_RETURN_NOT_OK(
686713
PyValue::Convert(this->primitive_type_, this->options_, value, view_));
@@ -721,6 +748,10 @@ class PyDictionaryConverter<U, enable_if_has_c_type<U>>
721748
Status Append(PyObject* value) override {
722749
if (PyValue::IsNull(this->options_, value)) {
723750
return this->value_builder_->AppendNull();
751+
} else if (arrow::py::is_scalar(value)) {
752+
ARROW_ASSIGN_OR_RAISE(std::shared_ptr<Scalar> scalar,
753+
arrow::py::unwrap_scalar(value));
754+
return this->value_builder_->AppendScalar(*scalar, 1);
724755
} else {
725756
ARROW_ASSIGN_OR_RAISE(auto converted,
726757
PyValue::Convert(this->value_type_, this->options_, value));
@@ -736,6 +767,10 @@ class PyDictionaryConverter<U, enable_if_has_string_view<U>>
736767
Status Append(PyObject* value) override {
737768
if (PyValue::IsNull(this->options_, value)) {
738769
return this->value_builder_->AppendNull();
770+
} else if (arrow::py::is_scalar(value)) {
771+
ARROW_ASSIGN_OR_RAISE(std::shared_ptr<Scalar> scalar,
772+
arrow::py::unwrap_scalar(value));
773+
return this->value_builder_->AppendScalar(*scalar, 1);
739774
} else {
740775
ARROW_RETURN_NOT_OK(
741776
PyValue::Convert(this->value_type_, this->options_, value, view_));
@@ -884,6 +919,10 @@ class PyStructConverter : public StructConverter<PyConverter, PyConverterTrait>
884919
Status Append(PyObject* value) override {
885920
if (PyValue::IsNull(this->options_, value)) {
886921
return this->struct_builder_->AppendNull();
922+
} else if (arrow::py::is_scalar(value)) {
923+
ARROW_ASSIGN_OR_RAISE(std::shared_ptr<Scalar> scalar,
924+
arrow::py::unwrap_scalar(value));
925+
return this->struct_builder_->AppendScalar(*scalar);
887926
}
888927
switch (input_kind_) {
889928
case InputKind::DICT:

‎python/pyarrow/tests/test_convert_builtin.py‎

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2363,3 +2363,141 @@ def test_array_from_pylist_offset_overflow():
23632363
assert isinstance(arr, pa.ChunkedArray)
23642364
assert len(arr) == 2**31
23652365
assert len(arr.chunks) > 1
2366+
2367+
2368+
@parametrize_with_collections_types
2369+
@pytest.mark.parametrize(('data', 'scalar_data', 'value_type'), [
2370+
([True, False, None], [pa.scalar(True), pa.scalar(False), None], pa.bool_()),
2371+
(
2372+
[1, 2, None],
2373+
[pa.scalar(1), pa.scalar(2), pa.scalar(None, pa.int64())],
2374+
pa.int64()
2375+
),
2376+
([1, None, None], [pa.scalar(1), None, pa.scalar(None, pa.int64())], pa.int64()),
2377+
([None, None], [pa.scalar(None), pa.scalar(None)], pa.null()),
2378+
([1., 2., None], [pa.scalar(1.), pa.scalar(2.), None], pa.float64()),
2379+
(
2380+
[None, datetime.date.today()],
2381+
[None, pa.scalar(datetime.date.today())],
2382+
pa.date32()
2383+
),
2384+
(
2385+
[None, datetime.date.today()],
2386+
[None, pa.scalar(datetime.date.today(), pa.date64())],
2387+
pa.date64()
2388+
),
2389+
(
2390+
[datetime.time(1, 1, 1), None],
2391+
[pa.scalar(datetime.time(1, 1, 1)), None],
2392+
pa.time64('us')
2393+
),
2394+
(
2395+
[datetime.timedelta(seconds=10)],
2396+
[pa.scalar(datetime.timedelta(seconds=10))],
2397+
pa.duration('us')
2398+
),
2399+
(
2400+
[None, datetime.datetime(2014, 1, 1)],
2401+
[None, pa.scalar(datetime.datetime(2014, 1, 1))],
2402+
pa.timestamp('us')
2403+
),
2404+
(
2405+
[pa.MonthDayNano([1, -1, -10100])],
2406+
[pa.scalar(pa.MonthDayNano([1, -1, -10100]))],
2407+
pa.month_day_nano_interval()
2408+
),
2409+
(["a", "b"], [pa.scalar("a"), pa.scalar("b")], pa.string()),
2410+
([b"a", b"b"], [pa.scalar(b"a"), pa.scalar(b"b")], pa.binary()),
2411+
(
2412+
[b"a", b"b"],
2413+
[pa.scalar(b"a", pa.binary(1)), pa.scalar(b"b", pa.binary(1))],
2414+
pa.binary(1)
2415+
),
2416+
([[1, 2, 3]], [pa.scalar([1, 2, 3])], pa.list_(pa.int64())),
2417+
([["a", "b"]], [pa.scalar(["a", "b"])], pa.list_(pa.string())),
2418+
(
2419+
[1, 2, None],
2420+
[pa.scalar(1, type=pa.int8()), pa.scalar(2, type=pa.int8()), None],
2421+
pa.int8()
2422+
),
2423+
([1, None], [pa.scalar(1.0, type=pa.int32()), None], pa.int32()),
2424+
(
2425+
["aaa", "bbb"],
2426+
[pa.scalar("aaa", type=pa.binary(3)), pa.scalar("bbb", type=pa.binary(3))],
2427+
pa.binary(3)),
2428+
([b"a"], [pa.scalar("a", type=pa.large_binary())], pa.large_binary()),
2429+
(["a"], [pa.scalar("a", type=pa.large_string())], pa.large_string()),
2430+
(
2431+
["a"],
2432+
[pa.scalar("a", type=pa.dictionary(pa.int64(), pa.string()))],
2433+
pa.dictionary(pa.int64(), pa.string())
2434+
),
2435+
(
2436+
["a", "b"],
2437+
[pa.scalar("a", pa.dictionary(pa.int64(), pa.string())),
2438+
pa.scalar("b", pa.dictionary(pa.int64(), pa.string()))],
2439+
pa.dictionary(pa.int64(), pa.string())
2440+
),
2441+
(
2442+
[1],
2443+
[pa.scalar(1, type=pa.dictionary(pa.int64(), pa.int32()))],
2444+
pa.dictionary(pa.int64(), pa.int32())
2445+
),
2446+
(
2447+
[(1, 2)],
2448+
[pa.scalar([('a', 1), ('b', 2)], type=pa.struct(
2449+
[('a', pa.int8()), ('b', pa.int8())]))],
2450+
pa.struct([('a', pa.int8()), ('b', pa.int8())])
2451+
),
2452+
(
2453+
[(1, 'bar')],
2454+
[pa.scalar([('a', 1), ('b', 'bar')], type=pa.struct(
2455+
[('a', pa.int8()), ('b', pa.string())]))],
2456+
pa.struct([('a', pa.int8()), ('b', pa.string())])
2457+
)
2458+
])
2459+
def test_array_accepts_pyarrow_scalar(seq, data, scalar_data, value_type):
2460+
if type(seq(scalar_data)) == set:
2461+
pytest.skip("The elements in the set get reordered.")
2462+
expect = pa.array(data, type=value_type)
2463+
result = pa.array(seq(scalar_data))
2464+
assert expect.equals(result)
2465+
2466+
result = pa.array(seq(scalar_data), type=value_type)
2467+
assert expect.equals(result)
2468+
2469+
2470+
@parametrize_with_collections_types
2471+
def test_array_accepts_pyarrow_scalar_errors(seq):
2472+
sequence = seq([pa.scalar(1), pa.scalar("a"), pa.scalar(3.0)])
2473+
with pytest.raises(pa.ArrowInvalid,
2474+
match="cannot mix scalars with different types"):
2475+
pa.array(sequence)
2476+
2477+
sequence = seq([1, pa.scalar("a"), None])
2478+
with pytest.raises(pa.ArrowInvalid,
2479+
match="pyarrow scalars cannot be mixed with other "
2480+
"Python scalar values currently"):
2481+
pa.array(sequence)
2482+
2483+
sequence = seq([np.float16("0.1"), pa.scalar("a"), None])
2484+
with pytest.raises(pa.ArrowInvalid,
2485+
match="pyarrow scalars cannot be mixed with other "
2486+
"Python scalar values currently"):
2487+
pa.array(sequence)
2488+
2489+
sequence = seq([pa.scalar("a"), np.float16("0.1"), None])
2490+
with pytest.raises(pa.ArrowInvalid,
2491+
match="pyarrow scalars cannot be mixed with other "
2492+
"Python scalar values currently"):
2493+
pa.array(sequence)
2494+
2495+
with pytest.raises(pa.ArrowInvalid,
2496+
match="Cannot append scalar of type string "
2497+
"to builder for type int32"):
2498+
pa.array([pa.scalar("a")], type=pa.int32())
2499+
2500+
with pytest.raises(pa.ArrowInvalid,
2501+
match="Cannot append scalar of type int64 "
2502+
"to builder for type null"):
2503+
pa.array([pa.scalar(1)], type=pa.null())

0 commit comments

Comments
 (0)