Skip to content

Commit 339e087

Browse files
authored
[fix](be) Reject incompatible Arrow arrays without crashing Stream Load (#69872)
### What problem does this PR solve? Issue Number: close #69858 Related PR: None Problem Summary: The shared Arrow BOOLEAN and DECIMAL SerDes dereference unchecked `dynamic_cast` results. Passing an `Int32Array` to the BOOLEAN reader or a `StringArray` to the DECIMAL64 reader can therefore terminate the BE instead of returning a load error. MAP and STRUCT have the same unchecked input-cast pattern. Reject incompatible arrays before accessing buffers, decimal metadata, or values, independently of `enable_arrow_input_validation`. Reject mismatched STRUCT field counts before indexing children. Add the column name and source/destination types to V1/V2 Arrow load errors. The current V2 Stream Load reader already binds supported input types with safe Arrow casts. Preserve those conversions, including valid integer-to-BOOLEAN and string-to-DECIMAL inputs, and test the shared SerDes directly for incompatible arrays.
1 parent fb61aee commit 339e087

47 files changed

Lines changed: 895 additions & 10 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎be/src/core/data_type_serde/data_type_decimal_serde.cpp‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -921,6 +921,11 @@ Status DataTypeDecimalSerDe<T>::read_column_from_arrow(IColumn& column,
921921
// Decimal<Int128I> for deicmalv3
922922
if constexpr (T == TYPE_DECIMALV2) {
923923
const auto* concrete_array = dynamic_cast<const arrow::DecimalArray*>(arrow_array);
924+
// A string or a different decimal width cannot be read as Decimal128, including NULLs.
925+
if (concrete_array == nullptr) {
926+
return Status::InvalidArgument("Expected Arrow Decimal128Array for {}, got {}",
927+
get_name(), arrow_array->type()->ToString());
928+
}
924929
const auto* arrow_decimal_type =
925930
static_cast<const arrow::DecimalType*>(arrow_array->type().get());
926931
const auto arrow_scale = arrow_decimal_type->scale();
@@ -948,13 +953,23 @@ Status DataTypeDecimalSerDe<T>::read_column_from_arrow(IColumn& column,
948953
}
949954
} else if constexpr (T == TYPE_DECIMAL32 || T == TYPE_DECIMAL64 || T == TYPE_DECIMAL128I) {
950955
const auto* concrete_array = dynamic_cast<const arrow::DecimalArray*>(arrow_array);
956+
// A string or a different decimal width cannot be read as Decimal128, including NULLs.
957+
if (concrete_array == nullptr) {
958+
return Status::InvalidArgument("Expected Arrow Decimal128Array for {}, got {}",
959+
get_name(), arrow_array->type()->ToString());
960+
}
951961
for (auto value_i = start; value_i < end; ++value_i) {
952962
const auto* value = concrete_array->Value(value_i);
953963
auto decimal_value = unaligned_load<FieldType>(value);
954964
column_data.emplace_back(decimal_value);
955965
}
956966
} else if constexpr (T == TYPE_DECIMAL256) {
957967
const auto* concrete_array = dynamic_cast<const arrow::Decimal256Array*>(arrow_array);
968+
// Reject incompatible arrays before Value(), independently of buffer validation.
969+
if (concrete_array == nullptr) {
970+
return Status::InvalidArgument("Expected Arrow Decimal256Array, got {}",
971+
arrow_array->type()->ToString());
972+
}
958973
for (auto value_i = start; value_i < end; ++value_i) {
959974
auto decimal_value = unaligned_load<FieldType>(concrete_array->Value(value_i));
960975
column_data.emplace_back(decimal_value);

‎be/src/core/data_type_serde/data_type_map_serde.cpp‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -549,6 +549,11 @@ Status DataTypeMapSerDe::read_column_from_arrow(IColumn& column, const arrow::Ar
549549
auto& column_map = static_cast<ColumnMap&>(column);
550550
auto& offsets_data = column_map.get_offsets();
551551
const auto* concrete_map = dynamic_cast<const arrow::MapArray*>(arrow_array);
552+
// A valid non-MAP Arrow column has no map offsets; reject it before accessing them.
553+
if (concrete_map == nullptr) {
554+
return Status::InvalidArgument("Expected Arrow MapArray, got {}",
555+
arrow_array->type()->ToString());
556+
}
552557
auto arrow_offsets_array = concrete_map->offsets();
553558
auto* arrow_offsets = dynamic_cast<arrow::Int32Array*>(arrow_offsets_array.get());
554559
if (config::enable_arrow_input_validation) {

‎be/src/core/data_type_serde/data_type_number_serde.cpp‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -956,6 +956,11 @@ Status DataTypeNumberSerDe<T>::read_column_from_arrow(IColumn& column,
956956
// now uint8 for bool
957957
if constexpr (T == TYPE_BOOLEAN) {
958958
const auto* concrete_array = dynamic_cast<const arrow::BooleanArray*>(arrow_array);
959+
// An Int32Array sent to a BOOLEAN column must fail even when buffer validation is off.
960+
if (concrete_array == nullptr) {
961+
return Status::InvalidArgument("Expected Arrow BooleanArray, got {}",
962+
arrow_array->type()->ToString());
963+
}
959964
if (config::enable_arrow_input_validation) {
960965
check_arrow_boolean_buffer(*concrete_array);
961966
}

‎be/src/core/data_type_serde/data_type_struct_serde.cpp‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -551,7 +551,16 @@ Status DataTypeStructSerDe::read_column_from_arrow(IColumn& column, const arrow:
551551
}
552552
auto& struct_column = static_cast<ColumnStruct&>(column);
553553
const auto* concrete_struct = dynamic_cast<const arrow::StructArray*>(arrow_array);
554-
DCHECK_EQ(struct_column.tuple_size(), concrete_struct->num_fields());
554+
// Reject non-STRUCT input before reading its fields, even with buffer validation disabled.
555+
if (concrete_struct == nullptr) {
556+
return Status::InvalidArgument("Expected Arrow StructArray, got {}",
557+
arrow_array->type()->ToString());
558+
}
559+
// A shorter input struct would otherwise index past its children; extra fields lose data.
560+
if (struct_column.tuple_size() != concrete_struct->num_fields()) {
561+
return Status::InvalidArgument("Expected Arrow StructArray with {} fields, got {}",
562+
struct_column.tuple_size(), concrete_struct->num_fields());
563+
}
555564
for (auto i = 0; i < struct_column.tuple_size(); ++i) {
556565
RETURN_IF_ERROR(elem_serdes_ptrs[i]->read_column_from_arrow(
557566
struct_column.get_column(i), concrete_struct->field(i).get(), start, end, ctz));

‎be/src/format/arrow/arrow_stream_reader.cpp‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -114,10 +114,15 @@ Status ArrowStreamReader::_do_get_next_block(Block* block, size_t* read_rows, bo
114114
column_name_in_block, column_name);
115115
}
116116

117-
RETURN_IF_ERROR(
118-
columns_guard.get_datatype_by_position(c)
119-
->get_serde()
120-
->read_column_from_arrow(*columns[c], column, 0, num_rows, _ctzz));
117+
const auto& data_type = columns_guard.get_datatype_by_position(c);
118+
auto status = data_type->get_serde()->read_column_from_arrow(*columns[c], column, 0,
119+
num_rows, _ctzz);
120+
// Type mismatches are load errors; retain their code and identify the input column.
121+
if (!status.ok()) {
122+
return status.prepend("Arrow column '" + column_name + "' (" +
123+
column->type()->ToString() + " -> " +
124+
data_type->get_name() + "): ");
125+
}
121126
} catch (Exception& e) {
122127
return Status::InternalError("Failed to convert from arrow to block: {}", e.what());
123128
}

‎be/src/format_v2/arrow/arrow_stream_reader.cpp‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -429,10 +429,16 @@ Status ArrowStreamReader::_prepare_arrays() {
429429
RETURN_IF_ERROR(normalize_arrow_array(_batch->column(_arrow_positions[id.value()]),
430430
_arrow_pool.get(), &array));
431431
arrow::compute::ExecContext context(_arrow_pool.get());
432-
std::shared_ptr<arrow::DataType> binding;
433-
RETURN_DORIS_STATUS_IF_RESULT_ERROR(
434-
binding, input_binding_type(array->type(), _arrow_types[id.value()],
435-
_file_slots[id.value()]->type()));
432+
auto binding_result = input_binding_type(array->type(), _arrow_types[id.value()],
433+
_file_slots[id.value()]->type());
434+
// A STRUCT field-count mismatch needs the same column context as a failed value cast.
435+
if (!binding_result.ok()) {
436+
return Status::InvalidArgument(
437+
"Cannot decode Arrow column '{}' ({} -> {}): {}",
438+
_file_slots[id.value()]->col_name(), array->type()->ToString(),
439+
_arrow_types[id.value()]->ToString(), binding_result.status().ToString());
440+
}
441+
auto binding = std::move(binding_result).ValueUnsafe();
436442
// Bind differing numeric widths before concrete SerDe array casts. Timestamp units
437443
// stay on the wire even inside containers, so Doris retains its own rounding semantics.
438444
if (!array->type()->Equals(binding)) {
@@ -443,8 +449,9 @@ Status ArrowStreamReader::_prepare_arrays() {
443449
options.allow_invalid_utf8 = true;
444450
auto converted = arrow::compute::Cast(array, binding, options, &context);
445451
if (!converted.ok()) {
446-
return Status::InvalidArgument("Cannot decode Arrow column '{}': {}",
452+
return Status::InvalidArgument("Cannot decode Arrow column '{}' ({} -> {}): {}",
447453
_file_slots[id.value()]->col_name(),
454+
array->type()->ToString(), binding->ToString(),
448455
converted.status().ToString());
449456
}
450457
array = converted->make_array();

‎be/test/core/data_type_serde/data_type_serde_arrow_validation_test.cpp‎

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,13 @@
3535
#include "core/column/column_nullable.h"
3636
#include "core/column/column_string.h"
3737
#include "core/column/column_varbinary.h"
38+
#include "core/data_type/data_type_array.h"
39+
#include "core/data_type/data_type_decimal.h"
40+
#include "core/data_type/data_type_map.h"
41+
#include "core/data_type/data_type_nullable.h"
42+
#include "core/data_type/data_type_number.h"
43+
#include "core/data_type/data_type_string.h"
44+
#include "core/data_type/data_type_struct.h"
3845
#include "core/data_type/primitive_type.h"
3946
#include "core/data_type_serde/data_type_array_serde.h"
4047
#include "core/data_type_serde/data_type_datev2_serde.h"
@@ -514,4 +521,142 @@ TEST(DataTypeSerDeArrowValidationTest, RejectsMapOffsetsBeyondKeysLength) {
514521
"map offsets beyond keys length should be rejected");
515522
}
516523

524+
// These arrays are well formed: the failure is their pairing with the destination type,
525+
// not a malformed buffer. Empty/all-NULL input must also reject the incompatible schema.
526+
TEST(DataTypeSerDeArrowValidationTest, RejectsIncompatibleArrayTypes) {
527+
const DataTypes types = {
528+
std::make_shared<DataTypeUInt8>(),
529+
std::make_shared<DataTypeDecimalV2>(27, 9),
530+
std::make_shared<DataTypeDecimal32>(9, 2),
531+
std::make_shared<DataTypeDecimal64>(18, 2),
532+
std::make_shared<DataTypeDecimal128>(38, 2),
533+
std::make_shared<DataTypeDecimal256>(76, 2),
534+
std::make_shared<DataTypeArray>(make_nullable(std::make_shared<DataTypeUInt8>())),
535+
std::make_shared<DataTypeMap>(make_nullable(std::make_shared<DataTypeString>()),
536+
make_nullable(std::make_shared<DataTypeUInt8>())),
537+
std::make_shared<DataTypeStruct>(
538+
DataTypes {make_nullable(std::make_shared<DataTypeUInt8>())}, Strings {"b"})};
539+
arrow::Int32Builder builder;
540+
ASSERT_TRUE(builder.Append(1).ok());
541+
std::shared_ptr<arrow::Array> non_null;
542+
ASSERT_TRUE(builder.Finish(&non_null).ok());
543+
for (const bool validation : {false, true}) {
544+
ScopedArrowInputValidation scoped_validation(validation);
545+
for (const auto& type : types) {
546+
for (const bool nullable : {false, true}) {
547+
const auto target = nullable ? make_nullable(type) : type;
548+
for (const auto length : {0, 1}) {
549+
const auto nulls = arrow::MakeArrayOfNull(arrow::int32(), length);
550+
ASSERT_TRUE(nulls.ok());
551+
for (const auto& input : {non_null, *nulls}) {
552+
SCOPED_TRACE(target->get_name() + ": " + input->ToString());
553+
auto column = target->create_column();
554+
const auto status = target->get_serde()->read_column_from_arrow(
555+
*column, input.get(), 0, input->length(), cctz::utc_time_zone());
556+
EXPECT_EQ(status.code(), ErrorCode::INVALID_ARGUMENT) << status;
557+
}
558+
}
559+
}
560+
}
561+
}
562+
}
563+
564+
TEST(DataTypeSerDeArrowValidationTest, RejectsStringAndWrongWidthDecimalArrays) {
565+
const DataTypes types = {
566+
std::make_shared<DataTypeDecimalV2>(27, 9), std::make_shared<DataTypeDecimal32>(9, 2),
567+
std::make_shared<DataTypeDecimal64>(18, 2), std::make_shared<DataTypeDecimal128>(38, 2),
568+
std::make_shared<DataTypeDecimal256>(76, 2)};
569+
arrow::StringBuilder builder;
570+
ASSERT_TRUE(builder.Append("1.50").ok());
571+
std::shared_ptr<arrow::Array> strings;
572+
ASSERT_TRUE(builder.Finish(&strings).ok());
573+
for (const bool validation : {false, true}) {
574+
ScopedArrowInputValidation scoped_validation(validation);
575+
for (const auto& type : types) {
576+
const auto wrong_width = type->get_primitive_type() == TYPE_DECIMAL256
577+
? arrow::decimal128(10, 2)
578+
: arrow::decimal256(76, 2);
579+
auto decimal = arrow::MakeArrayOfNull(wrong_width, 1);
580+
ASSERT_TRUE(decimal.ok());
581+
for (const bool nullable : {false, true}) {
582+
const auto target = nullable ? make_nullable(type) : type;
583+
for (const auto& input : {strings, *decimal}) {
584+
auto column = target->create_column();
585+
const auto status = target->get_serde()->read_column_from_arrow(
586+
*column, input.get(), 0, input->length(), cctz::utc_time_zone());
587+
EXPECT_EQ(status.code(), ErrorCode::INVALID_ARGUMENT) << status;
588+
}
589+
}
590+
}
591+
}
592+
}
593+
594+
TEST(DataTypeSerDeArrowValidationTest, RejectsStructFieldCountMismatch) {
595+
const auto boolean = make_nullable(std::make_shared<DataTypeUInt8>());
596+
const auto type = std::make_shared<DataTypeStruct>(DataTypes {boolean}, Strings {"b"});
597+
for (const bool validation : {false, true}) {
598+
ScopedArrowInputValidation scoped_validation(validation);
599+
for (const auto& arrow_type :
600+
{arrow::struct_({}), arrow::struct_({arrow::field("b", arrow::boolean()),
601+
arrow::field("extra", arrow::boolean())})}) {
602+
auto input = arrow::MakeArrayOfNull(arrow_type, 1);
603+
ASSERT_TRUE(input.ok());
604+
auto column = type->create_column();
605+
const auto status = type->get_serde()->read_column_from_arrow(*column, input->get(), 0,
606+
1, cctz::utc_time_zone());
607+
EXPECT_EQ(status.code(), ErrorCode::INVALID_ARGUMENT) << status;
608+
}
609+
}
610+
}
611+
612+
TEST(DataTypeSerDeArrowValidationTest, RejectsIncompatibleNestedArrayTypes) {
613+
const auto boolean = make_nullable(std::make_shared<DataTypeUInt8>());
614+
const DataTypes targets = {
615+
std::make_shared<DataTypeArray>(boolean),
616+
std::make_shared<DataTypeMap>(make_nullable(std::make_shared<DataTypeString>()),
617+
boolean),
618+
std::make_shared<DataTypeStruct>(DataTypes {boolean}, Strings {"b"})};
619+
const std::vector<std::shared_ptr<arrow::DataType>> inputs = {
620+
arrow::list(arrow::int32()), arrow::map(arrow::utf8(), arrow::int32()),
621+
arrow::struct_({arrow::field("b", arrow::int32())})};
622+
for (const bool validation : {false, true}) {
623+
ScopedArrowInputValidation scoped_validation(validation);
624+
for (size_t i = 0; i < targets.size(); ++i) {
625+
auto input = arrow::MakeArrayOfNull(inputs[i], 1);
626+
ASSERT_TRUE(input.ok());
627+
auto column = targets[i]->create_column();
628+
const auto status = targets[i]->get_serde()->read_column_from_arrow(
629+
*column, input->get(), 0, 1, cctz::utc_time_zone());
630+
EXPECT_EQ(status.code(), ErrorCode::INVALID_ARGUMENT) << status;
631+
}
632+
}
633+
}
634+
635+
TEST(DataTypeSerDeArrowValidationTest, AcceptsMatchingArrayTypesWithValidationOnAndOff) {
636+
const DataTypes targets = {std::make_shared<DataTypeUInt8>(),
637+
std::make_shared<DataTypeDecimalV2>(27, 9),
638+
std::make_shared<DataTypeDecimal32>(9, 2),
639+
std::make_shared<DataTypeDecimal64>(18, 2),
640+
std::make_shared<DataTypeDecimal128>(38, 2),
641+
std::make_shared<DataTypeDecimal256>(76, 2)};
642+
const std::vector<std::shared_ptr<arrow::DataType>> inputs = {
643+
arrow::boolean(), arrow::decimal128(27, 9), arrow::decimal128(9, 2),
644+
arrow::decimal128(18, 2), arrow::decimal128(38, 2), arrow::decimal256(76, 2)};
645+
for (const bool validation : {false, true}) {
646+
ScopedArrowInputValidation scoped_validation(validation);
647+
for (size_t i = 0; i < targets.size(); ++i) {
648+
const auto target = make_nullable(targets[i]);
649+
for (const auto length : {0, 2}) {
650+
auto input = arrow::MakeArrayOfNull(inputs[i], length);
651+
ASSERT_TRUE(input.ok());
652+
auto column = target->create_column();
653+
const auto status = target->get_serde()->read_column_from_arrow(
654+
*column, input->get(), 0, length, cctz::utc_time_zone());
655+
EXPECT_TRUE(status.ok()) << status;
656+
EXPECT_EQ(column->size(), length);
657+
}
658+
}
659+
}
660+
}
661+
517662
} // namespace doris

‎be/test/format_v2/table/load_reader_test.cpp‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1238,6 +1238,63 @@ TEST_F(LoadArrowPipeTest, NumericIntStillRejectsAnOverflowingUInt32) {
12381238
ASSERT_TRUE(reader.close().ok());
12391239
}
12401240

1241+
TEST_F(LoadArrowPipeTest, ConvertsTheReportedInt32BooleanInput) {
1242+
set_column_type(make_nullable(std::make_shared<DataTypeUInt8>()));
1243+
arrow::Int32Builder builder;
1244+
ASSERT_TRUE(builder.AppendValues({1, 0}).ok());
1245+
auto values = builder.Finish();
1246+
ASSERT_TRUE(values.ok());
1247+
append_batch(arrow::RecordBatch::Make(
1248+
arrow::schema({arrow::field("raw_value", arrow::int32())}), 2, {*values}));
1249+
ASSERT_TRUE(_pipe->finish().ok());
1250+
LoadTableReader reader(source.get(), destination.get());
1251+
ASSERT_TRUE(prepare(reader, FileFormat::ARROW).ok());
1252+
Block output(destination->slots(), 0);
1253+
bool eos = false;
1254+
const auto status = reader.get_block(&output, &eos);
1255+
ASSERT_TRUE(status.ok()) << status;
1256+
ASSERT_EQ(output.rows(), 2);
1257+
const auto& column = assert_cast<const ColumnNullable&>(*output.get_by_position(0).column);
1258+
EXPECT_EQ(column.get_nested_column().get_int(0), 1);
1259+
EXPECT_EQ(column.get_nested_column().get_int(1), 0);
1260+
ASSERT_TRUE(reader.close().ok());
1261+
}
1262+
1263+
TEST_F(LoadArrowPipeTest, InvalidBooleanInputIncludesTheColumnAndBothTypes) {
1264+
set_column_type(make_nullable(std::make_shared<DataTypeUInt8>()));
1265+
append_stream({"not-a-boolean"}, "raw_value");
1266+
ASSERT_TRUE(_pipe->finish().ok());
1267+
LoadTableReader reader(source.get(), destination.get());
1268+
ASSERT_TRUE(prepare(reader, FileFormat::ARROW).ok());
1269+
Block output(destination->slots(), 0);
1270+
bool eos = false;
1271+
const auto status = reader.get_block(&output, &eos);
1272+
EXPECT_EQ(status.code(), ErrorCode::INVALID_ARGUMENT) << status;
1273+
EXPECT_NE(status.to_string().find("Arrow column 'raw_value' (string -> bool)"),
1274+
std::string::npos)
1275+
<< status;
1276+
ASSERT_TRUE(reader.close().ok());
1277+
}
1278+
1279+
TEST_F(LoadArrowPipeTest, StructFieldCountErrorIncludesTheColumnAndBothTypes) {
1280+
set_column_type(make_nullable(std::make_shared<DataTypeStruct>(
1281+
DataTypes {make_nullable(std::make_shared<DataTypeUInt8>())}, Strings {"b"})));
1282+
auto input = arrow::MakeArrayOfNull(arrow::struct_({}), 1);
1283+
ASSERT_TRUE(input.ok());
1284+
append_batch(arrow::RecordBatch::Make(
1285+
arrow::schema({arrow::field("raw_value", (*input)->type())}), 1, {*input}));
1286+
ASSERT_TRUE(_pipe->finish().ok());
1287+
LoadTableReader reader(source.get(), destination.get());
1288+
ASSERT_TRUE(prepare(reader, FileFormat::ARROW).ok());
1289+
Block output(destination->slots(), 0);
1290+
bool eos = false;
1291+
const auto status = reader.get_block(&output, &eos);
1292+
EXPECT_EQ(status.code(), ErrorCode::INVALID_ARGUMENT) << status;
1293+
EXPECT_NE(status.to_string().find("Arrow column 'raw_value'"), std::string::npos) << status;
1294+
EXPECT_NE(status.to_string().find("struct<> -> struct<b: bool>"), std::string::npos) << status;
1295+
ASSERT_TRUE(reader.close().ok());
1296+
}
1297+
12411298
TEST_F(LoadArrowPipeTest, ReadsCompressedIpcBatches) {
12421299
arrow::StringBuilder builder;
12431300
ASSERT_TRUE(builder.Append("compressed").ok());
Binary file not shown.
Binary file not shown.

0 commit comments

Comments
 (0)