|
35 | 35 | #include "core/column/column_nullable.h" |
36 | 36 | #include "core/column/column_string.h" |
37 | 37 | #include "core/column/column_varbinary.h" |
| 38 | +#include "core/data_type/data_type_array.h" |
| 39 | +#include "core/data_type/data_type_decimal.h" |
| 40 | +#include "core/data_type/data_type_map.h" |
| 41 | +#include "core/data_type/data_type_nullable.h" |
| 42 | +#include "core/data_type/data_type_number.h" |
| 43 | +#include "core/data_type/data_type_string.h" |
| 44 | +#include "core/data_type/data_type_struct.h" |
38 | 45 | #include "core/data_type/primitive_type.h" |
39 | 46 | #include "core/data_type_serde/data_type_array_serde.h" |
40 | 47 | #include "core/data_type_serde/data_type_datev2_serde.h" |
@@ -514,4 +521,142 @@ TEST(DataTypeSerDeArrowValidationTest, RejectsMapOffsetsBeyondKeysLength) { |
514 | 521 | "map offsets beyond keys length should be rejected"); |
515 | 522 | } |
516 | 523 |
|
| 524 | +// These arrays are well formed: the failure is their pairing with the destination type, |
| 525 | +// not a malformed buffer. Empty/all-NULL input must also reject the incompatible schema. |
| 526 | +TEST(DataTypeSerDeArrowValidationTest, RejectsIncompatibleArrayTypes) { |
| 527 | + const DataTypes types = { |
| 528 | + std::make_shared<DataTypeUInt8>(), |
| 529 | + std::make_shared<DataTypeDecimalV2>(27, 9), |
| 530 | + std::make_shared<DataTypeDecimal32>(9, 2), |
| 531 | + std::make_shared<DataTypeDecimal64>(18, 2), |
| 532 | + std::make_shared<DataTypeDecimal128>(38, 2), |
| 533 | + std::make_shared<DataTypeDecimal256>(76, 2), |
| 534 | + std::make_shared<DataTypeArray>(make_nullable(std::make_shared<DataTypeUInt8>())), |
| 535 | + std::make_shared<DataTypeMap>(make_nullable(std::make_shared<DataTypeString>()), |
| 536 | + make_nullable(std::make_shared<DataTypeUInt8>())), |
| 537 | + std::make_shared<DataTypeStruct>( |
| 538 | + DataTypes {make_nullable(std::make_shared<DataTypeUInt8>())}, Strings {"b"})}; |
| 539 | + arrow::Int32Builder builder; |
| 540 | + ASSERT_TRUE(builder.Append(1).ok()); |
| 541 | + std::shared_ptr<arrow::Array> non_null; |
| 542 | + ASSERT_TRUE(builder.Finish(&non_null).ok()); |
| 543 | + for (const bool validation : {false, true}) { |
| 544 | + ScopedArrowInputValidation scoped_validation(validation); |
| 545 | + for (const auto& type : types) { |
| 546 | + for (const bool nullable : {false, true}) { |
| 547 | + const auto target = nullable ? make_nullable(type) : type; |
| 548 | + for (const auto length : {0, 1}) { |
| 549 | + const auto nulls = arrow::MakeArrayOfNull(arrow::int32(), length); |
| 550 | + ASSERT_TRUE(nulls.ok()); |
| 551 | + for (const auto& input : {non_null, *nulls}) { |
| 552 | + SCOPED_TRACE(target->get_name() + ": " + input->ToString()); |
| 553 | + auto column = target->create_column(); |
| 554 | + const auto status = target->get_serde()->read_column_from_arrow( |
| 555 | + *column, input.get(), 0, input->length(), cctz::utc_time_zone()); |
| 556 | + EXPECT_EQ(status.code(), ErrorCode::INVALID_ARGUMENT) << status; |
| 557 | + } |
| 558 | + } |
| 559 | + } |
| 560 | + } |
| 561 | + } |
| 562 | +} |
| 563 | + |
| 564 | +TEST(DataTypeSerDeArrowValidationTest, RejectsStringAndWrongWidthDecimalArrays) { |
| 565 | + const DataTypes types = { |
| 566 | + std::make_shared<DataTypeDecimalV2>(27, 9), std::make_shared<DataTypeDecimal32>(9, 2), |
| 567 | + std::make_shared<DataTypeDecimal64>(18, 2), std::make_shared<DataTypeDecimal128>(38, 2), |
| 568 | + std::make_shared<DataTypeDecimal256>(76, 2)}; |
| 569 | + arrow::StringBuilder builder; |
| 570 | + ASSERT_TRUE(builder.Append("1.50").ok()); |
| 571 | + std::shared_ptr<arrow::Array> strings; |
| 572 | + ASSERT_TRUE(builder.Finish(&strings).ok()); |
| 573 | + for (const bool validation : {false, true}) { |
| 574 | + ScopedArrowInputValidation scoped_validation(validation); |
| 575 | + for (const auto& type : types) { |
| 576 | + const auto wrong_width = type->get_primitive_type() == TYPE_DECIMAL256 |
| 577 | + ? arrow::decimal128(10, 2) |
| 578 | + : arrow::decimal256(76, 2); |
| 579 | + auto decimal = arrow::MakeArrayOfNull(wrong_width, 1); |
| 580 | + ASSERT_TRUE(decimal.ok()); |
| 581 | + for (const bool nullable : {false, true}) { |
| 582 | + const auto target = nullable ? make_nullable(type) : type; |
| 583 | + for (const auto& input : {strings, *decimal}) { |
| 584 | + auto column = target->create_column(); |
| 585 | + const auto status = target->get_serde()->read_column_from_arrow( |
| 586 | + *column, input.get(), 0, input->length(), cctz::utc_time_zone()); |
| 587 | + EXPECT_EQ(status.code(), ErrorCode::INVALID_ARGUMENT) << status; |
| 588 | + } |
| 589 | + } |
| 590 | + } |
| 591 | + } |
| 592 | +} |
| 593 | + |
| 594 | +TEST(DataTypeSerDeArrowValidationTest, RejectsStructFieldCountMismatch) { |
| 595 | + const auto boolean = make_nullable(std::make_shared<DataTypeUInt8>()); |
| 596 | + const auto type = std::make_shared<DataTypeStruct>(DataTypes {boolean}, Strings {"b"}); |
| 597 | + for (const bool validation : {false, true}) { |
| 598 | + ScopedArrowInputValidation scoped_validation(validation); |
| 599 | + for (const auto& arrow_type : |
| 600 | + {arrow::struct_({}), arrow::struct_({arrow::field("b", arrow::boolean()), |
| 601 | + arrow::field("extra", arrow::boolean())})}) { |
| 602 | + auto input = arrow::MakeArrayOfNull(arrow_type, 1); |
| 603 | + ASSERT_TRUE(input.ok()); |
| 604 | + auto column = type->create_column(); |
| 605 | + const auto status = type->get_serde()->read_column_from_arrow(*column, input->get(), 0, |
| 606 | + 1, cctz::utc_time_zone()); |
| 607 | + EXPECT_EQ(status.code(), ErrorCode::INVALID_ARGUMENT) << status; |
| 608 | + } |
| 609 | + } |
| 610 | +} |
| 611 | + |
| 612 | +TEST(DataTypeSerDeArrowValidationTest, RejectsIncompatibleNestedArrayTypes) { |
| 613 | + const auto boolean = make_nullable(std::make_shared<DataTypeUInt8>()); |
| 614 | + const DataTypes targets = { |
| 615 | + std::make_shared<DataTypeArray>(boolean), |
| 616 | + std::make_shared<DataTypeMap>(make_nullable(std::make_shared<DataTypeString>()), |
| 617 | + boolean), |
| 618 | + std::make_shared<DataTypeStruct>(DataTypes {boolean}, Strings {"b"})}; |
| 619 | + const std::vector<std::shared_ptr<arrow::DataType>> inputs = { |
| 620 | + arrow::list(arrow::int32()), arrow::map(arrow::utf8(), arrow::int32()), |
| 621 | + arrow::struct_({arrow::field("b", arrow::int32())})}; |
| 622 | + for (const bool validation : {false, true}) { |
| 623 | + ScopedArrowInputValidation scoped_validation(validation); |
| 624 | + for (size_t i = 0; i < targets.size(); ++i) { |
| 625 | + auto input = arrow::MakeArrayOfNull(inputs[i], 1); |
| 626 | + ASSERT_TRUE(input.ok()); |
| 627 | + auto column = targets[i]->create_column(); |
| 628 | + const auto status = targets[i]->get_serde()->read_column_from_arrow( |
| 629 | + *column, input->get(), 0, 1, cctz::utc_time_zone()); |
| 630 | + EXPECT_EQ(status.code(), ErrorCode::INVALID_ARGUMENT) << status; |
| 631 | + } |
| 632 | + } |
| 633 | +} |
| 634 | + |
| 635 | +TEST(DataTypeSerDeArrowValidationTest, AcceptsMatchingArrayTypesWithValidationOnAndOff) { |
| 636 | + const DataTypes targets = {std::make_shared<DataTypeUInt8>(), |
| 637 | + std::make_shared<DataTypeDecimalV2>(27, 9), |
| 638 | + std::make_shared<DataTypeDecimal32>(9, 2), |
| 639 | + std::make_shared<DataTypeDecimal64>(18, 2), |
| 640 | + std::make_shared<DataTypeDecimal128>(38, 2), |
| 641 | + std::make_shared<DataTypeDecimal256>(76, 2)}; |
| 642 | + const std::vector<std::shared_ptr<arrow::DataType>> inputs = { |
| 643 | + arrow::boolean(), arrow::decimal128(27, 9), arrow::decimal128(9, 2), |
| 644 | + arrow::decimal128(18, 2), arrow::decimal128(38, 2), arrow::decimal256(76, 2)}; |
| 645 | + for (const bool validation : {false, true}) { |
| 646 | + ScopedArrowInputValidation scoped_validation(validation); |
| 647 | + for (size_t i = 0; i < targets.size(); ++i) { |
| 648 | + const auto target = make_nullable(targets[i]); |
| 649 | + for (const auto length : {0, 2}) { |
| 650 | + auto input = arrow::MakeArrayOfNull(inputs[i], length); |
| 651 | + ASSERT_TRUE(input.ok()); |
| 652 | + auto column = target->create_column(); |
| 653 | + const auto status = target->get_serde()->read_column_from_arrow( |
| 654 | + *column, input->get(), 0, length, cctz::utc_time_zone()); |
| 655 | + EXPECT_TRUE(status.ok()) << status; |
| 656 | + EXPECT_EQ(column->size(), length); |
| 657 | + } |
| 658 | + } |
| 659 | + } |
| 660 | +} |
| 661 | + |
517 | 662 | } // namespace doris |
0 commit comments