Skip to content

Commit 6a47e4d

Browse files
NickCrewspitrou
andauthored
GH-33592: [C++] support casting nullable fields to non-nullable if there are no null values (#43782)
* GitHub Issue: #33592 Notes for myself/fixer: - [tests that need to get updated](https://github.com/search?q=repo%3Aapache%2Farrow%20%22cannot%20cast%20nullable%20field%22&type=code) (almost definitely not a complete list) - [update: actually we should handle the go implementation in the go repository.] hmm, looks like [go wrapper does its own nullability checks](https://github.com/apache/arrow/blob/f078942ce2df68de8f48c3b4233132133601ca53/go/arrow/compute/cast.go#L283-L286). I assume this is just an optimization to not have to go into the C++ and hit the error down there. So if we just delete this check then I think the C++ logic will handle all of it??? - [how the plain column handles this cast](https://github.com/apache/arrow/blob/f078942ce2df68de8f48c3b4233132133601ca53/python/pyarrow/table.pxi#L3238-L3243), some logic like this probably needs to get ported over to the struct implementation - (running from /cpp/build) `cmake .. --preset ninja-debug-basic`, then `cmake --build . && PYTHON=python ctest -R 'arrow-compute-scalar-cast-test' --output-on-failure` to run the specific test - to format: `uvx pre-commit run --all-files clang-format` Lead-authored-by: Nick Crews <[email protected]> Co-authored-by: Antoine Pitrou <[email protected]> Signed-off-by: Antoine Pitrou <[email protected]>
1 parent 251758d commit 6a47e4d

2 files changed

Lines changed: 90 additions & 81 deletions

File tree

‎cpp/src/arrow/compute/kernels/scalar_cast_nested.cc‎

Lines changed: 18 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -363,10 +363,6 @@ struct CastStruct {
363363
const auto& in_field = in_type.field(in_field_index);
364364
// If there are more in_fields check if they match the out_field.
365365
if (in_field->name() == out_field->name()) {
366-
if (in_field->nullable() && !out_field->nullable()) {
367-
return Status::TypeError("cannot cast nullable field to non-nullable field: ",
368-
in_type.ToString(), " ", out_type.ToString());
369-
}
370366
// Found matching in_field and out_field.
371367
fields_to_select[out_field_index++] = in_field_index;
372368
// Using the same in_field for multiple out_fields is not allowed.
@@ -403,17 +399,27 @@ struct CastStruct {
403399
}
404400

405401
int out_field_index = 0;
406-
for (int field_index : fields_to_select) {
407-
const auto& target_type = out->type()->field(out_field_index++)->type();
408-
if (field_index == kFillNullSentinel) {
409-
ARROW_ASSIGN_OR_RAISE(auto nulls,
410-
MakeArrayOfNull(target_type->GetSharedPtr(), batch.length));
402+
for (int in_field_index : fields_to_select) {
403+
const auto& out_field = out_type.field(out_field_index++);
404+
const auto& out_field_type = out_field->type();
405+
if (in_field_index == kFillNullSentinel) {
406+
ARROW_ASSIGN_OR_RAISE(
407+
auto nulls, MakeArrayOfNull(out_field_type->GetSharedPtr(), batch.length,
408+
ctx->memory_pool()));
411409
out_array->child_data.push_back(nulls->data());
412410
} else {
413-
const auto& values = (in_array.child_data[field_index].ToArrayData()->Slice(
411+
const auto& in_field = in_type.field(in_field_index);
412+
const auto& in_values = (in_array.child_data[in_field_index].ToArrayData()->Slice(
414413
in_array.offset, in_array.length));
415-
ARROW_ASSIGN_OR_RAISE(Datum cast_values,
416-
Cast(values, target_type, options, ctx->exec_context()));
414+
if (in_field->nullable() && !out_field->nullable() &&
415+
in_values->GetNullCount() > 0) {
416+
return Status::Invalid(
417+
"field '", in_field->name(), "' of type ", in_field->type()->ToString(),
418+
" has nulls. Can't cast to non-nullable field '", out_field->name(),
419+
"' of type ", out_field_type->ToString());
420+
}
421+
ARROW_ASSIGN_OR_RAISE(Datum cast_values, Cast(in_values, out_field_type, options,
422+
ctx->exec_context()));
417423
DCHECK(cast_values.is_array());
418424
out_array->child_data.push_back(cast_values.array());
419425
}

‎cpp/src/arrow/compute/kernels/scalar_cast_test.cc‎

Lines changed: 72 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -4076,92 +4076,95 @@ TEST(Cast, StructToBiggerNullableStruct) {
40764076
TEST(Cast, StructToDifferentNullabilityStruct) {
40774077
{
40784078
// OK to go from non-nullable to nullable...
4079-
std::vector<std::shared_ptr<Field>> fields_src_non_nullable = {
4079+
std::vector<std::shared_ptr<Field>> fields_src = {
40804080
std::make_shared<Field>("a", int8(), false),
40814081
std::make_shared<Field>("b", int8(), false),
40824082
std::make_shared<Field>("c", int8(), false)};
4083-
std::shared_ptr<Array> a_src_non_nullable, b_src_non_nullable, c_src_non_nullable;
4084-
a_src_non_nullable = ArrayFromJSON(int8(), "[11, 23, 56]");
4085-
b_src_non_nullable = ArrayFromJSON(int8(), "[32, 46, 37]");
4086-
c_src_non_nullable = ArrayFromJSON(int8(), "[95, 11, 44]");
4087-
ASSERT_OK_AND_ASSIGN(
4088-
auto src_non_nullable,
4089-
StructArray::Make({a_src_non_nullable, b_src_non_nullable, c_src_non_nullable},
4090-
fields_src_non_nullable));
4091-
4092-
std::shared_ptr<Array> a_dest_nullable, b_dest_nullable, c_dest_nullable;
4093-
a_dest_nullable = ArrayFromJSON(int64(), "[11, 23, 56]");
4094-
b_dest_nullable = ArrayFromJSON(int64(), "[32, 46, 37]");
4095-
c_dest_nullable = ArrayFromJSON(int64(), "[95, 11, 44]");
4096-
4097-
std::vector<std::shared_ptr<Field>> fields_dest1_nullable = {
4083+
std::vector<std::shared_ptr<Array>> arrays_src = {
4084+
ArrayFromJSON(int8(), "[11, 23, 56]"),
4085+
ArrayFromJSON(int8(), "[32, 46, 37]"),
4086+
ArrayFromJSON(int8(), "[95, 11, 44]"),
4087+
};
4088+
ASSERT_OK_AND_ASSIGN(auto src, StructArray::Make(arrays_src, fields_src));
4089+
4090+
std::vector<std::shared_ptr<Field>> fields_dest = {
40984091
std::make_shared<Field>("a", int64(), true),
40994092
std::make_shared<Field>("b", int64(), true),
4100-
std::make_shared<Field>("c", int64(), true)};
4101-
ASSERT_OK_AND_ASSIGN(
4102-
auto dest1_nullable,
4103-
StructArray::Make({a_dest_nullable, b_dest_nullable, c_dest_nullable},
4104-
fields_dest1_nullable));
4105-
CheckCast(src_non_nullable, dest1_nullable);
4106-
4107-
std::vector<std::shared_ptr<Field>> fields_dest2_nullable = {
4108-
std::make_shared<Field>("a", int64(), true),
4109-
std::make_shared<Field>("c", int64(), true)};
4110-
ASSERT_OK_AND_ASSIGN(
4111-
auto dest2_nullable,
4112-
StructArray::Make({a_dest_nullable, c_dest_nullable}, fields_dest2_nullable));
4113-
CheckCast(src_non_nullable, dest2_nullable);
4114-
4115-
std::vector<std::shared_ptr<Field>> fields_dest3_nullable = {
4116-
std::make_shared<Field>("b", int64(), true)};
4117-
ASSERT_OK_AND_ASSIGN(auto dest3_nullable,
4118-
StructArray::Make({b_dest_nullable}, fields_dest3_nullable));
4119-
CheckCast(src_non_nullable, dest3_nullable);
4093+
std::make_shared<Field>("c", int64(), true),
4094+
};
4095+
std::vector<std::shared_ptr<Array>> arrays_dest = {
4096+
ArrayFromJSON(int64(), "[11, 23, 56]"),
4097+
ArrayFromJSON(int64(), "[32, 46, 37]"),
4098+
ArrayFromJSON(int64(), "[95, 11, 44]"),
4099+
};
4100+
ASSERT_OK_AND_ASSIGN(auto dest, StructArray::Make(arrays_dest, fields_dest));
4101+
CheckCast(src, dest);
4102+
4103+
std::vector<std::shared_ptr<Field>> fields_dest_ac = {fields_dest[0], fields_dest[2]};
4104+
std::vector<std::shared_ptr<Array>> arrays_dest_ac = {arrays_dest[0], arrays_dest[2]};
4105+
ASSERT_OK_AND_ASSIGN(auto dest_ac, StructArray::Make(arrays_dest_ac, fields_dest_ac));
4106+
CheckCast(src, dest_ac);
4107+
4108+
std::vector<std::shared_ptr<Field>> fields_dest_b = {fields_dest[1]};
4109+
std::vector<std::shared_ptr<Array>> arrays_dest_b = {arrays_dest[1]};
4110+
ASSERT_OK_AND_ASSIGN(auto dest_b, StructArray::Make(arrays_dest_b, fields_dest_b));
4111+
CheckCast(src, dest_b);
41204112
}
41214113
{
4122-
// But NOT OK to go from nullable to non-nullable...
4123-
std::vector<std::shared_ptr<Field>> fields_src_nullable = {
4114+
// But when going from nullable to non-nullable, all data must be non-null...
4115+
std::vector<std::shared_ptr<Field>> fields_src = {
41244116
std::make_shared<Field>("a", int8(), true),
41254117
std::make_shared<Field>("b", int8(), true),
41264118
std::make_shared<Field>("c", int8(), true)};
4127-
std::shared_ptr<Array> a_src_nullable, b_src_nullable, c_src_nullable;
4128-
a_src_nullable = ArrayFromJSON(int8(), "[1, null, 5]");
4129-
b_src_nullable = ArrayFromJSON(int8(), "[3, 4, null]");
4130-
c_src_nullable = ArrayFromJSON(int8(), "[9, 11, 44]");
4131-
ASSERT_OK_AND_ASSIGN(
4132-
auto src_nullable,
4133-
StructArray::Make({a_src_nullable, b_src_nullable, c_src_nullable},
4134-
fields_src_nullable));
4135-
4136-
std::vector<std::shared_ptr<Field>> fields_dest1_non_nullable = {
4119+
std::vector<std::shared_ptr<Array>> arrays_src = {
4120+
ArrayFromJSON(int8(), "[1, null, 5]"),
4121+
ArrayFromJSON(int8(), "[3, 4, null]"),
4122+
ArrayFromJSON(int8(), "[9, 11, 44]"),
4123+
};
4124+
ASSERT_OK_AND_ASSIGN(auto src, StructArray::Make(arrays_src, fields_src));
4125+
4126+
std::vector<std::shared_ptr<Field>> fields_dest = {
41374127
std::make_shared<Field>("a", int64(), false),
41384128
std::make_shared<Field>("b", int64(), false),
41394129
std::make_shared<Field>("c", int64(), false)};
4140-
const auto dest1_non_nullable = arrow::struct_(fields_dest1_non_nullable);
4141-
const auto options1_non_nullable = CastOptions::Safe(dest1_non_nullable);
41424130
EXPECT_RAISES_WITH_MESSAGE_THAT(
4143-
TypeError,
4144-
::testing::HasSubstr("cannot cast nullable field to non-nullable field"),
4145-
Cast(src_nullable, options1_non_nullable));
4131+
Invalid,
4132+
::testing::HasSubstr(
4133+
"field 'a' of type int8 has nulls. Can't cast to non-nullable field 'a' "
4134+
"of type int64"),
4135+
Cast(src, CastOptions::Safe(arrow::struct_(fields_dest))));
41464136

4147-
std::vector<std::shared_ptr<Field>> fields_dest2_non_nullable = {
4148-
std::make_shared<Field>("a", int64(), false),
4149-
std::make_shared<Field>("c", int64(), false)};
4150-
const auto dest2_non_nullable = arrow::struct_(fields_dest2_non_nullable);
4151-
const auto options2_non_nullable = CastOptions::Safe(dest2_non_nullable);
4137+
std::vector<std::shared_ptr<Field>> fields_dest_ac = {fields_dest[0], fields_dest[2]};
41524138
EXPECT_RAISES_WITH_MESSAGE_THAT(
4153-
TypeError,
4154-
::testing::HasSubstr("cannot cast nullable field to non-nullable field"),
4155-
Cast(src_nullable, options2_non_nullable));
4156-
4157-
std::vector<std::shared_ptr<Field>> fields_dest3_non_nullable = {
4158-
std::make_shared<Field>("c", int64(), false)};
4159-
const auto dest3_non_nullable = arrow::struct_(fields_dest3_non_nullable);
4160-
const auto options3_non_nullable = CastOptions::Safe(dest3_non_nullable);
4139+
Invalid,
4140+
::testing::HasSubstr(
4141+
"field 'a' of type int8 has nulls. Can't cast to non-nullable field 'a' "
4142+
"of type int64"),
4143+
Cast(src, CastOptions::Safe(arrow::struct_(fields_dest_ac))));
4144+
4145+
// if we only select a field with no nulls, it should be fine:
4146+
std::vector<std::shared_ptr<Field>> fields_dest_c = {fields_dest[2]};
4147+
std::vector<std::shared_ptr<Array>> arrays_dest_c = {
4148+
ArrayFromJSON(int64(), "[9, 11, 44]")};
4149+
ASSERT_OK_AND_ASSIGN(auto dest_c, StructArray::Make(arrays_dest_c, fields_dest_c));
4150+
CheckCast(src, dest_c);
4151+
4152+
// A slice that doesn't contain nulls is castable...
4153+
std::vector<std::shared_ptr<Array>> arrays_dest_0 = {
4154+
ArrayFromJSON(int64(), "[1]"),
4155+
ArrayFromJSON(int64(), "[3]"),
4156+
ArrayFromJSON(int64(), "[9]"),
4157+
};
4158+
ASSERT_OK_AND_ASSIGN(auto dest_0, StructArray::Make(arrays_dest_0, fields_dest));
4159+
CheckCast(src->Slice(0, 1), dest_0);
4160+
4161+
// ...but a slice that contains nulls will error.
41614162
EXPECT_RAISES_WITH_MESSAGE_THAT(
4162-
TypeError,
4163-
::testing::HasSubstr("cannot cast nullable field to non-nullable field"),
4164-
Cast(src_nullable, options3_non_nullable));
4163+
Invalid,
4164+
::testing::HasSubstr(
4165+
"field 'a' of type int8 has nulls. Can't cast to non-nullable field 'a' "
4166+
"of type int64"),
4167+
Cast(src->Slice(1, 3), CastOptions::Safe(arrow::struct_(fields_dest))));
41654168
}
41664169
}
41674170

0 commit comments

Comments
 (0)