|
31 | 31 |
|
32 | 32 | #include "core/column/column_array.h" |
33 | 33 | #include "core/column/column_complex.h" |
| 34 | +#include "core/column/column_decimal.h" |
34 | 35 | #include "core/column/column_fixed_length_object.h" |
35 | 36 | #include "core/column/column_map.h" |
36 | 37 | #include "core/column/column_nullable.h" |
|
41 | 42 | #include "core/data_type/data_type_array.h" |
42 | 43 | #include "core/data_type/data_type_bitmap.h" |
43 | 44 | #include "core/data_type/data_type_date_or_datetime_v2.h" |
| 45 | +#include "core/data_type/data_type_decimal.h" |
44 | 46 | #include "core/data_type/data_type_ipv4.h" |
45 | 47 | #include "core/data_type/data_type_map.h" |
46 | 48 | #include "core/data_type/data_type_nullable.h" |
@@ -720,8 +722,188 @@ class LoadArrowPipeTest : public LoadJsonPipeTest { |
720 | 722 | } |
721 | 723 | ASSERT_TRUE(reader.close().ok()); |
722 | 724 | } |
| 725 | + void check_null_fixed_list_struct(bool narrow, bool all_null = false, int width = 1) { |
| 726 | + const auto structure = make_nullable(std::make_shared<DataTypeStruct>( |
| 727 | + DataTypes {make_nullable(std::make_shared<DataTypeInt32>())}, Strings {"a"})); |
| 728 | + set_column_type(make_nullable(std::make_shared<DataTypeArray>(structure))); |
| 729 | + const auto wire_type = narrow ? arrow::int64() : arrow::int32(); |
| 730 | + std::unique_ptr<arrow::ArrayBuilder> builder; |
| 731 | + ASSERT_TRUE(arrow::MakeBuilder(arrow::default_memory_pool(), wire_type, &builder).ok()); |
| 732 | + for (int i = 0; i < 2 * width; ++i) { |
| 733 | + const auto status = narrow ? static_cast<arrow::Int64Builder&>(*builder).Append( |
| 734 | + i < width ? std::numeric_limits<int64_t>::max() |
| 735 | + : 7 + i - width) |
| 736 | + : static_cast<arrow::Int32Builder&>(*builder).Append( |
| 737 | + i < width ? 1 + i : 7 + i - width); |
| 738 | + ASSERT_TRUE(status.ok()) << status; |
| 739 | + } |
| 740 | + auto values = builder->Finish(); |
| 741 | + ASSERT_TRUE(values.ok()); |
| 742 | + auto structs = arrow::StructArray::Make( |
| 743 | + {*values}, arrow::FieldVector {arrow::field("a", wire_type, false)}); |
| 744 | + ASSERT_TRUE(structs.ok()); |
| 745 | + arrow::BooleanBuilder validity; |
| 746 | + ASSERT_TRUE(validity.AppendValues(std::vector<bool> {false, !all_null}).ok()); |
| 747 | + auto valid = validity.Finish(); |
| 748 | + ASSERT_TRUE(valid.ok()); |
| 749 | + auto lists = std::make_shared<arrow::FixedSizeListArray>( |
| 750 | + arrow::fixed_size_list((*structs)->type(), width), 2, *structs, |
| 751 | + (*valid)->data()->buffers[1], all_null ? 2 : 1); |
| 752 | + append_batch(arrow::RecordBatch::Make( |
| 753 | + arrow::schema({arrow::field("raw_value", lists->type())}), 2, {lists})); |
| 754 | + ASSERT_TRUE(_pipe->finish().ok()); |
| 755 | + LoadTableReader reader(source.get(), destination.get()); |
| 756 | + ASSERT_TRUE(prepare(reader, FileFormat::ARROW).ok()); |
| 757 | + Block output(destination->slots(), 0); |
| 758 | + bool eos = false; |
| 759 | + const auto status = reader.get_block(&output, &eos); |
| 760 | + ASSERT_TRUE(status.ok()) << status; |
| 761 | + ASSERT_EQ(output.rows(), 2); |
| 762 | + const auto& outer = assert_cast<const ColumnNullable&>(*output.get_by_position(0).column); |
| 763 | + EXPECT_TRUE(outer.is_null_at(0)); |
| 764 | + EXPECT_EQ(outer.is_null_at(1), all_null); |
| 765 | + const auto& array = assert_cast<const ColumnArray&>(outer.get_nested_column()); |
| 766 | + EXPECT_EQ(array.get_offsets()[0], 0); |
| 767 | + EXPECT_EQ(array.get_offsets()[1], all_null ? 0 : width); |
| 768 | + const auto& elements = assert_cast<const ColumnNullable&>(array.get_data()); |
| 769 | + const auto& fields = assert_cast<const ColumnStruct&>(elements.get_nested_column()); |
| 770 | + if (!all_null) { |
| 771 | + const auto& numbers = assert_cast<const ColumnNullable&>(fields.get_column(0)); |
| 772 | + for (int i = 0; i < width; ++i) { |
| 773 | + EXPECT_FALSE(numbers.is_null_at(i)); |
| 774 | + EXPECT_EQ(numbers.get_nested_column().get_int(i), 7 + i); |
| 775 | + } |
| 776 | + } |
| 777 | + ASSERT_TRUE(reader.close().ok()); |
| 778 | + } |
| 779 | + void check_null_fixed_list_variant(bool extension) { |
| 780 | + set_column_type(make_nullable(std::make_shared<DataTypeArray>( |
| 781 | + make_nullable(std::make_shared<DataTypeVariantV2>())))); |
| 782 | + std::shared_ptr<arrow::Array> values; |
| 783 | + make_native_variant_array(extension, &values); |
| 784 | + ASSERT_NE(values, nullptr); |
| 785 | + arrow::BooleanBuilder validity; |
| 786 | + ASSERT_TRUE(validity.AppendValues(std::vector<bool> {true, false, true}).ok()); |
| 787 | + auto valid = validity.Finish(); |
| 788 | + ASSERT_TRUE(valid.ok()); |
| 789 | + auto lists = std::make_shared<arrow::FixedSizeListArray>( |
| 790 | + arrow::fixed_size_list(values->type(), 1), 3, values, (*valid)->data()->buffers[1], |
| 791 | + 1); |
| 792 | + append_batch(arrow::RecordBatch::Make( |
| 793 | + arrow::schema({arrow::field("raw_value", lists->type())}), 3, {lists})); |
| 794 | + ASSERT_TRUE(_pipe->finish().ok()); |
| 795 | + LoadTableReader reader(source.get(), destination.get()); |
| 796 | + ASSERT_TRUE(prepare(reader, FileFormat::ARROW).ok()); |
| 797 | + Block output(destination->slots(), 0); |
| 798 | + bool eos = false; |
| 799 | + const auto status = reader.get_block(&output, &eos); |
| 800 | + ASSERT_TRUE(status.ok()) << status; |
| 801 | + ASSERT_EQ(output.rows(), 3); |
| 802 | + const auto& outer = assert_cast<const ColumnNullable&>(*output.get_by_position(0).column); |
| 803 | + EXPECT_TRUE(outer.is_null_at(1)); |
| 804 | + const auto& array = assert_cast<const ColumnArray&>(outer.get_nested_column()); |
| 805 | + EXPECT_EQ(array.get_offsets()[0], 1); |
| 806 | + EXPECT_EQ(array.get_offsets()[1], 1); |
| 807 | + EXPECT_EQ(array.get_offsets()[2], 2); |
| 808 | + const auto& elements = assert_cast<const ColumnNullable&>(array.get_data()); |
| 809 | + const auto& variants = assert_cast<const ColumnVariantV2&>(elements.get_nested_column()); |
| 810 | + EXPECT_EQ(variants.get_value_ref(0).get_int(), 42); |
| 811 | + EXPECT_TRUE(elements.is_null_at(1)); |
| 812 | + ASSERT_TRUE(reader.close().ok()); |
| 813 | + } |
| 814 | + void check_decimal_rescaling(bool legacy, bool nested = false, int scale = 10) { |
| 815 | + const auto decimal_type = |
| 816 | + make_nullable(legacy ? DataTypePtr(std::make_shared<DataTypeDecimalV2>(27, 9)) |
| 817 | + : DataTypePtr(std::make_shared<DataTypeDecimal128>(27, 9))); |
| 818 | + set_column_type(nested ? make_nullable(std::make_shared<DataTypeArray>(decimal_type)) |
| 819 | + : decimal_type); |
| 820 | + const int64_t unscaled = scale == 10 ? 12345678901LL : 1234567; |
| 821 | + arrow::Decimal128Builder builder(arrow::decimal128(20, scale), |
| 822 | + arrow::default_memory_pool()); |
| 823 | + ASSERT_TRUE(builder.Append(arrow::Decimal128(unscaled)).ok()); |
| 824 | + ASSERT_TRUE(builder.Append(arrow::Decimal128(-unscaled)).ok()); |
| 825 | + ASSERT_TRUE(builder.AppendNull().ok()); |
| 826 | + auto decimals = builder.Finish(); |
| 827 | + ASSERT_TRUE(decimals.ok()); |
| 828 | + std::shared_ptr<arrow::Array> values = *decimals; |
| 829 | + if (nested) { |
| 830 | + arrow::Int32Builder offsets; |
| 831 | + ASSERT_TRUE(offsets.AppendValues({0, 3}).ok()); |
| 832 | + auto positions = offsets.Finish(); |
| 833 | + ASSERT_TRUE(positions.ok()); |
| 834 | + auto lists = arrow::ListArray::FromArrays(**positions, *values); |
| 835 | + ASSERT_TRUE(lists.ok()); |
| 836 | + values = *lists; |
| 837 | + } |
| 838 | + append_batch( |
| 839 | + arrow::RecordBatch::Make(arrow::schema({arrow::field("raw_value", values->type())}), |
| 840 | + nested ? 1 : 3, {values})); |
| 841 | + ASSERT_TRUE(_pipe->finish().ok()); |
| 842 | + LoadTableReader reader(source.get(), destination.get()); |
| 843 | + ASSERT_TRUE(prepare(reader, FileFormat::ARROW).ok()); |
| 844 | + Block output(destination->slots(), 0); |
| 845 | + bool eos = false; |
| 846 | + const auto status = reader.get_block(&output, &eos); |
| 847 | + if (!legacy) { |
| 848 | + EXPECT_FALSE(status.ok()); |
| 849 | + EXPECT_NE(status.to_string().find("Cannot decode Arrow column"), std::string::npos) |
| 850 | + << status; |
| 851 | + } else { |
| 852 | + ASSERT_TRUE(status.ok()) << status; |
| 853 | + ASSERT_EQ(output.rows(), nested ? 1 : 3); |
| 854 | + const IColumn* result = output.get_by_position(0).column.get(); |
| 855 | + if (nested) { |
| 856 | + const auto& outer = assert_cast<const ColumnNullable&>(*result).get_nested_column(); |
| 857 | + result = &assert_cast<const ColumnArray&>(outer).get_data(); |
| 858 | + } |
| 859 | + const auto& nullable = assert_cast<const ColumnNullable&>(*result); |
| 860 | + const auto& numbers = |
| 861 | + assert_cast<const ColumnDecimal<TYPE_DECIMALV2>&>(nullable.get_nested_column()); |
| 862 | + const int64_t expected = scale == 10 ? 1234567890 : 1234567000; |
| 863 | + EXPECT_EQ(numbers.get_element(0).value(), expected); |
| 864 | + EXPECT_EQ(numbers.get_element(1).value(), -expected); |
| 865 | + EXPECT_TRUE(nullable.is_null_at(2)); |
| 866 | + } |
| 867 | + ASSERT_TRUE(reader.close().ok()); |
| 868 | + } |
723 | 869 | }; |
724 | 870 |
|
| 871 | +TEST_F(LoadArrowPipeTest, NullFixedSizeListPreservesNonNullableStructDescendants) { |
| 872 | + check_null_fixed_list_struct(false); |
| 873 | +} |
| 874 | + |
| 875 | +TEST_F(LoadArrowPipeTest, NullFixedSizeListCompactsStructsBeforeCheckedNarrowing) { |
| 876 | + check_null_fixed_list_struct(true, false, 2); |
| 877 | +} |
| 878 | + |
| 879 | +TEST_F(LoadArrowPipeTest, AllNullFixedSizeListCompactsNonNullableStructDescendants) { |
| 880 | + check_null_fixed_list_struct(false, true); |
| 881 | +} |
| 882 | + |
| 883 | +TEST_F(LoadArrowPipeTest, NullFixedSizeListPreservesNativeVariantStorage) { |
| 884 | + check_null_fixed_list_variant(false); |
| 885 | +} |
| 886 | + |
| 887 | +TEST_F(LoadArrowPipeTest, NullFixedSizeListPreservesNativeVariantExtension) { |
| 888 | + check_null_fixed_list_variant(true); |
| 889 | +} |
| 890 | + |
| 891 | +TEST_F(LoadArrowPipeTest, DecimalV2RetainsHigherScaleInputTruncation) { |
| 892 | + check_decimal_rescaling(true); |
| 893 | +} |
| 894 | + |
| 895 | +TEST_F(LoadArrowPipeTest, NestedDecimalV2RetainsHigherScaleInputTruncation) { |
| 896 | + check_decimal_rescaling(true, true); |
| 897 | +} |
| 898 | + |
| 899 | +TEST_F(LoadArrowPipeTest, DecimalV2RetainsLowerScaleInputExpansion) { |
| 900 | + check_decimal_rescaling(true, false, 6); |
| 901 | +} |
| 902 | + |
| 903 | +TEST_F(LoadArrowPipeTest, DecimalV3StillRejectsLossyScaleReduction) { |
| 904 | + check_decimal_rescaling(false); |
| 905 | +} |
| 906 | + |
725 | 907 | TEST_F(LoadArrowPipeTest, NativeVariantStoragePreservesValuesAndSqlNulls) { |
726 | 908 | check_native_variant(false); |
727 | 909 | } |
|
0 commit comments