Skip to content

Commit 213c279

Browse files
committed
[fix](load) Preserve Arrow fixed-list and DECIMALV2 semantics
### What problem does this PR solve? Related PR: #69668 Problem Summary: Taking nullable indices from a fixed-size list synthesized NULL descendants under non-nullable STRUCT fields. View NULL fixed lists with 64-bit variable offsets before compaction so hidden child ranges are dropped as a whole, while valid child fields and native VARIANT carriers remain unchanged. Preserve decimal128 wire scales for DECIMALV2 input so its existing SerDe can truncate extra fractional digits or expand lower scales to nine. Other decimal types retain safe Arrow scale checks. ### Release note Restore accepted Arrow loads containing NULL fixed-size lists with non-nullable descendants and DECIMALV2 values requiring scale conversion. ### Check List (For Author) - Test: 221 ASAN BE unit tests, six regression suites, production BE build, clang-format 16, clang-tidy, and build hygiene checks. - Behavior changed: Preserve existing input semantics for NULL fixed lists and DECIMALV2 rescaling without weakening other numeric conversions. - Does this need documentation? No; existing load interfaces are retained.
1 parent d7bd729 commit 213c279

2 files changed

Lines changed: 210 additions & 1 deletion

File tree

‎be/src/format_v2/arrow/arrow_stream_reader.cpp‎

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,11 @@ arrow::Result<std::shared_ptr<arrow::DataType>> input_binding_type(
102102
if (type->get_primitive_type() == TYPE_VARIANT) {
103103
return wire;
104104
}
105+
// DECIMALV2 SerDe rescales decimal128 input to scale nine, truncating extra fractional
106+
// digits toward zero (1.2345678901 -> 1.234567890). Other decimals retain checked casts.
107+
if (type->get_primitive_type() == TYPE_DECIMALV2 && wire->id() == arrow::Type::DECIMAL128) {
108+
return wire;
109+
}
105110
// IPv4 is a 32-bit bit pattern, including addresses above INT32_MAX. Numeric INT columns
106111
// still use the checked conversion below instead of inheriting this storage exception.
107112
if (type->get_primitive_type() == TYPE_IPV4 && wire->id() == arrow::Type::UINT32) {
@@ -170,19 +175,41 @@ arrow::Result<std::shared_ptr<arrow::Array>> mask_struct_children(
170175
return arrow::MakeArray(data);
171176
}
172177

178+
arrow::Result<std::shared_ptr<arrow::Array>> fixed_list_as_large_list(
179+
const arrow::FixedSizeListArray& list, const std::shared_ptr<arrow::Buffer>& validity,
180+
arrow::MemoryPool* pool) {
181+
arrow::Int64Builder offsets(pool);
182+
ARROW_RETURN_NOT_OK(offsets.Reserve(list.length() + 1));
183+
for (int64_t i = 0; i <= list.length(); ++i) {
184+
ARROW_RETURN_NOT_OK(offsets.Append(list.value_offset(i)));
185+
}
186+
ARROW_ASSIGN_OR_RAISE(auto values, offsets.Finish());
187+
return std::make_shared<arrow::LargeListArray>(
188+
arrow::large_list(list.list_type()->value_field()), list.length(),
189+
values->data()->buffers[1], list.values(), validity, list.null_count());
190+
}
191+
173192
arrow::Result<std::shared_ptr<arrow::Array>> compact_null_containers(
174193
std::shared_ptr<arrow::Array> array, arrow::compute::ExecContext* context) {
175194
if (array->null_count() == 0) {
176195
return array;
177196
}
178197
// NULL lists/maps can reference nonempty child ranges. Take drops those ranges without
179-
// making map keys nullable, and masks the hidden children of fixed-size lists.
198+
// making map keys or other non-nullable descendants nullable.
180199
arrow::Int64Builder indices(context->memory_pool());
181200
ARROW_RETURN_NOT_OK(indices.Reserve(array->length()));
182201
for (int64_t i = 0; i < array->length(); ++i) {
183202
ARROW_RETURN_NOT_OK(array->IsNull(i) ? indices.AppendNull() : indices.Append(i));
184203
}
185204
ARROW_ASSIGN_OR_RAISE(auto positions, indices.Finish());
205+
// Fixed-list Take propagates a NULL index into every child, even non-nullable STRUCT
206+
// fields such as a:INT NOT NULL. View it as a variable list first so Take drops the whole
207+
// hidden range instead. Keep 64-bit offsets until the final checked LIST cast.
208+
if (array->type_id() == arrow::Type::FIXED_SIZE_LIST) {
209+
ARROW_ASSIGN_OR_RAISE(array, fixed_list_as_large_list(
210+
static_cast<const arrow::FixedSizeListArray&>(*array),
211+
positions->null_bitmap(), context->memory_pool()));
212+
}
186213
ARROW_ASSIGN_OR_RAISE(
187214
auto compacted,
188215
arrow::compute::Take(array, positions, arrow::compute::TakeOptions::NoBoundsCheck(),

‎be/test/format_v2/table/load_reader_test.cpp‎

Lines changed: 182 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131

3232
#include "core/column/column_array.h"
3333
#include "core/column/column_complex.h"
34+
#include "core/column/column_decimal.h"
3435
#include "core/column/column_fixed_length_object.h"
3536
#include "core/column/column_map.h"
3637
#include "core/column/column_nullable.h"
@@ -41,6 +42,7 @@
4142
#include "core/data_type/data_type_array.h"
4243
#include "core/data_type/data_type_bitmap.h"
4344
#include "core/data_type/data_type_date_or_datetime_v2.h"
45+
#include "core/data_type/data_type_decimal.h"
4446
#include "core/data_type/data_type_ipv4.h"
4547
#include "core/data_type/data_type_map.h"
4648
#include "core/data_type/data_type_nullable.h"
@@ -720,8 +722,188 @@ class LoadArrowPipeTest : public LoadJsonPipeTest {
720722
}
721723
ASSERT_TRUE(reader.close().ok());
722724
}
725+
void check_null_fixed_list_struct(bool narrow, bool all_null = false, int width = 1) {
726+
const auto structure = make_nullable(std::make_shared<DataTypeStruct>(
727+
DataTypes {make_nullable(std::make_shared<DataTypeInt32>())}, Strings {"a"}));
728+
set_column_type(make_nullable(std::make_shared<DataTypeArray>(structure)));
729+
const auto wire_type = narrow ? arrow::int64() : arrow::int32();
730+
std::unique_ptr<arrow::ArrayBuilder> builder;
731+
ASSERT_TRUE(arrow::MakeBuilder(arrow::default_memory_pool(), wire_type, &builder).ok());
732+
for (int i = 0; i < 2 * width; ++i) {
733+
const auto status = narrow ? static_cast<arrow::Int64Builder&>(*builder).Append(
734+
i < width ? std::numeric_limits<int64_t>::max()
735+
: 7 + i - width)
736+
: static_cast<arrow::Int32Builder&>(*builder).Append(
737+
i < width ? 1 + i : 7 + i - width);
738+
ASSERT_TRUE(status.ok()) << status;
739+
}
740+
auto values = builder->Finish();
741+
ASSERT_TRUE(values.ok());
742+
auto structs = arrow::StructArray::Make(
743+
{*values}, arrow::FieldVector {arrow::field("a", wire_type, false)});
744+
ASSERT_TRUE(structs.ok());
745+
arrow::BooleanBuilder validity;
746+
ASSERT_TRUE(validity.AppendValues(std::vector<bool> {false, !all_null}).ok());
747+
auto valid = validity.Finish();
748+
ASSERT_TRUE(valid.ok());
749+
auto lists = std::make_shared<arrow::FixedSizeListArray>(
750+
arrow::fixed_size_list((*structs)->type(), width), 2, *structs,
751+
(*valid)->data()->buffers[1], all_null ? 2 : 1);
752+
append_batch(arrow::RecordBatch::Make(
753+
arrow::schema({arrow::field("raw_value", lists->type())}), 2, {lists}));
754+
ASSERT_TRUE(_pipe->finish().ok());
755+
LoadTableReader reader(source.get(), destination.get());
756+
ASSERT_TRUE(prepare(reader, FileFormat::ARROW).ok());
757+
Block output(destination->slots(), 0);
758+
bool eos = false;
759+
const auto status = reader.get_block(&output, &eos);
760+
ASSERT_TRUE(status.ok()) << status;
761+
ASSERT_EQ(output.rows(), 2);
762+
const auto& outer = assert_cast<const ColumnNullable&>(*output.get_by_position(0).column);
763+
EXPECT_TRUE(outer.is_null_at(0));
764+
EXPECT_EQ(outer.is_null_at(1), all_null);
765+
const auto& array = assert_cast<const ColumnArray&>(outer.get_nested_column());
766+
EXPECT_EQ(array.get_offsets()[0], 0);
767+
EXPECT_EQ(array.get_offsets()[1], all_null ? 0 : width);
768+
const auto& elements = assert_cast<const ColumnNullable&>(array.get_data());
769+
const auto& fields = assert_cast<const ColumnStruct&>(elements.get_nested_column());
770+
if (!all_null) {
771+
const auto& numbers = assert_cast<const ColumnNullable&>(fields.get_column(0));
772+
for (int i = 0; i < width; ++i) {
773+
EXPECT_FALSE(numbers.is_null_at(i));
774+
EXPECT_EQ(numbers.get_nested_column().get_int(i), 7 + i);
775+
}
776+
}
777+
ASSERT_TRUE(reader.close().ok());
778+
}
779+
void check_null_fixed_list_variant(bool extension) {
780+
set_column_type(make_nullable(std::make_shared<DataTypeArray>(
781+
make_nullable(std::make_shared<DataTypeVariantV2>()))));
782+
std::shared_ptr<arrow::Array> values;
783+
make_native_variant_array(extension, &values);
784+
ASSERT_NE(values, nullptr);
785+
arrow::BooleanBuilder validity;
786+
ASSERT_TRUE(validity.AppendValues(std::vector<bool> {true, false, true}).ok());
787+
auto valid = validity.Finish();
788+
ASSERT_TRUE(valid.ok());
789+
auto lists = std::make_shared<arrow::FixedSizeListArray>(
790+
arrow::fixed_size_list(values->type(), 1), 3, values, (*valid)->data()->buffers[1],
791+
1);
792+
append_batch(arrow::RecordBatch::Make(
793+
arrow::schema({arrow::field("raw_value", lists->type())}), 3, {lists}));
794+
ASSERT_TRUE(_pipe->finish().ok());
795+
LoadTableReader reader(source.get(), destination.get());
796+
ASSERT_TRUE(prepare(reader, FileFormat::ARROW).ok());
797+
Block output(destination->slots(), 0);
798+
bool eos = false;
799+
const auto status = reader.get_block(&output, &eos);
800+
ASSERT_TRUE(status.ok()) << status;
801+
ASSERT_EQ(output.rows(), 3);
802+
const auto& outer = assert_cast<const ColumnNullable&>(*output.get_by_position(0).column);
803+
EXPECT_TRUE(outer.is_null_at(1));
804+
const auto& array = assert_cast<const ColumnArray&>(outer.get_nested_column());
805+
EXPECT_EQ(array.get_offsets()[0], 1);
806+
EXPECT_EQ(array.get_offsets()[1], 1);
807+
EXPECT_EQ(array.get_offsets()[2], 2);
808+
const auto& elements = assert_cast<const ColumnNullable&>(array.get_data());
809+
const auto& variants = assert_cast<const ColumnVariantV2&>(elements.get_nested_column());
810+
EXPECT_EQ(variants.get_value_ref(0).get_int(), 42);
811+
EXPECT_TRUE(elements.is_null_at(1));
812+
ASSERT_TRUE(reader.close().ok());
813+
}
814+
void check_decimal_rescaling(bool legacy, bool nested = false, int scale = 10) {
815+
const auto decimal_type =
816+
make_nullable(legacy ? DataTypePtr(std::make_shared<DataTypeDecimalV2>(27, 9))
817+
: DataTypePtr(std::make_shared<DataTypeDecimal128>(27, 9)));
818+
set_column_type(nested ? make_nullable(std::make_shared<DataTypeArray>(decimal_type))
819+
: decimal_type);
820+
const int64_t unscaled = scale == 10 ? 12345678901LL : 1234567;
821+
arrow::Decimal128Builder builder(arrow::decimal128(20, scale),
822+
arrow::default_memory_pool());
823+
ASSERT_TRUE(builder.Append(arrow::Decimal128(unscaled)).ok());
824+
ASSERT_TRUE(builder.Append(arrow::Decimal128(-unscaled)).ok());
825+
ASSERT_TRUE(builder.AppendNull().ok());
826+
auto decimals = builder.Finish();
827+
ASSERT_TRUE(decimals.ok());
828+
std::shared_ptr<arrow::Array> values = *decimals;
829+
if (nested) {
830+
arrow::Int32Builder offsets;
831+
ASSERT_TRUE(offsets.AppendValues({0, 3}).ok());
832+
auto positions = offsets.Finish();
833+
ASSERT_TRUE(positions.ok());
834+
auto lists = arrow::ListArray::FromArrays(**positions, *values);
835+
ASSERT_TRUE(lists.ok());
836+
values = *lists;
837+
}
838+
append_batch(
839+
arrow::RecordBatch::Make(arrow::schema({arrow::field("raw_value", values->type())}),
840+
nested ? 1 : 3, {values}));
841+
ASSERT_TRUE(_pipe->finish().ok());
842+
LoadTableReader reader(source.get(), destination.get());
843+
ASSERT_TRUE(prepare(reader, FileFormat::ARROW).ok());
844+
Block output(destination->slots(), 0);
845+
bool eos = false;
846+
const auto status = reader.get_block(&output, &eos);
847+
if (!legacy) {
848+
EXPECT_FALSE(status.ok());
849+
EXPECT_NE(status.to_string().find("Cannot decode Arrow column"), std::string::npos)
850+
<< status;
851+
} else {
852+
ASSERT_TRUE(status.ok()) << status;
853+
ASSERT_EQ(output.rows(), nested ? 1 : 3);
854+
const IColumn* result = output.get_by_position(0).column.get();
855+
if (nested) {
856+
const auto& outer = assert_cast<const ColumnNullable&>(*result).get_nested_column();
857+
result = &assert_cast<const ColumnArray&>(outer).get_data();
858+
}
859+
const auto& nullable = assert_cast<const ColumnNullable&>(*result);
860+
const auto& numbers =
861+
assert_cast<const ColumnDecimal<TYPE_DECIMALV2>&>(nullable.get_nested_column());
862+
const int64_t expected = scale == 10 ? 1234567890 : 1234567000;
863+
EXPECT_EQ(numbers.get_element(0).value(), expected);
864+
EXPECT_EQ(numbers.get_element(1).value(), -expected);
865+
EXPECT_TRUE(nullable.is_null_at(2));
866+
}
867+
ASSERT_TRUE(reader.close().ok());
868+
}
723869
};
724870

871+
TEST_F(LoadArrowPipeTest, NullFixedSizeListPreservesNonNullableStructDescendants) {
872+
check_null_fixed_list_struct(false);
873+
}
874+
875+
TEST_F(LoadArrowPipeTest, NullFixedSizeListCompactsStructsBeforeCheckedNarrowing) {
876+
check_null_fixed_list_struct(true, false, 2);
877+
}
878+
879+
TEST_F(LoadArrowPipeTest, AllNullFixedSizeListCompactsNonNullableStructDescendants) {
880+
check_null_fixed_list_struct(false, true);
881+
}
882+
883+
TEST_F(LoadArrowPipeTest, NullFixedSizeListPreservesNativeVariantStorage) {
884+
check_null_fixed_list_variant(false);
885+
}
886+
887+
TEST_F(LoadArrowPipeTest, NullFixedSizeListPreservesNativeVariantExtension) {
888+
check_null_fixed_list_variant(true);
889+
}
890+
891+
TEST_F(LoadArrowPipeTest, DecimalV2RetainsHigherScaleInputTruncation) {
892+
check_decimal_rescaling(true);
893+
}
894+
895+
TEST_F(LoadArrowPipeTest, NestedDecimalV2RetainsHigherScaleInputTruncation) {
896+
check_decimal_rescaling(true, true);
897+
}
898+
899+
TEST_F(LoadArrowPipeTest, DecimalV2RetainsLowerScaleInputExpansion) {
900+
check_decimal_rescaling(true, false, 6);
901+
}
902+
903+
TEST_F(LoadArrowPipeTest, DecimalV3StillRejectsLossyScaleReduction) {
904+
check_decimal_rescaling(false);
905+
}
906+
725907
TEST_F(LoadArrowPipeTest, NativeVariantStoragePreservesValuesAndSqlNulls) {
726908
check_native_variant(false);
727909
}

0 commit comments

Comments
 (0)