|
17 | 17 |
|
18 | 18 | #include "storage/segment/variant/binary_column_extract_iterator.h" |
19 | 19 |
|
| 20 | +#include <glog/logging.h> |
20 | 21 | #include <gtest/gtest.h> |
21 | 22 |
|
22 | 23 | #include <algorithm> |
|
40 | 41 | #include "core/value/jsonb_value.h" |
41 | 42 | #include "exec/common/variant_util.h" |
42 | 43 | #include "storage/iterators.h" |
| 44 | +#include "storage/segment/variant/sparse_column_merge_iterator.h" |
| 45 | +#include "storage/segment/variant/v2/variant_assembler.h" |
43 | 46 |
|
44 | 47 | namespace doris::segment_v2 { |
45 | 48 | namespace { |
@@ -789,5 +792,94 @@ TEST(BinaryColumnExtractIteratorV2Test, DoesNotRescanFrozenPathOrdering) { |
789 | 792 | EXPECT_EQ(variant_v2_json_at(variant, 0), "7"); |
790 | 793 | } |
791 | 794 |
|
| 795 | +// Debug builds must reject a sparse row that violates the byte-order invariant; the check is a |
| 796 | +// DCHECK so release scans and compaction pay nothing. |
| 797 | +#if DCHECK_IS_ON() |
| 798 | +TEST(VariantAssemblerTest, DebugCheckRejectsUnsortedSparseRows) { |
| 799 | + auto sparse = variant_util::create_variant_binary_column(); |
| 800 | + auto& map = assert_cast<ColumnMap&>(*sparse); |
| 801 | + DataTypeInt64 int_type; |
| 802 | + auto ints = ColumnInt64::create(); |
| 803 | + ints->insert_value(1); |
| 804 | + ints->insert_value(2); |
| 805 | + append_storage_cell(map, "a.c", int_type, *ints, 0); |
| 806 | + append_storage_cell(map, "a-b", int_type, *ints, 1); |
| 807 | + map.get_offsets().push_back(2); |
| 808 | + |
| 809 | + variant_v2::VariantAssemblerOptions options; |
| 810 | + options.storage_map_kind = variant_v2::StorageMapKind::SPARSE; |
| 811 | + auto assembler = variant_v2::VariantAssembler::create(std::move(options)); |
| 812 | + ASSERT_TRUE(assembler.has_value()); |
| 813 | + EXPECT_DEATH( |
| 814 | + { |
| 815 | + ColumnNullable::MutablePtr result; |
| 816 | + (void)(*assembler) |
| 817 | + ->assemble({.num_rows = map.size(), |
| 818 | + .materialized_columns = {}, |
| 819 | + .storage_map = &map}, |
| 820 | + &result); |
| 821 | + }, |
| 822 | + "is_variant_sparse_row_sorted"); |
| 823 | +} |
| 824 | +#endif |
| 825 | + |
| 826 | +// Vertical compaction merges the source sparse row with materialized subcolumns. The interleaving |
| 827 | +// assumes both sides are byte-ordered, and the merged row is what later readers see, so it must |
| 828 | +// keep that invariant. |
| 829 | +TEST(SparseColumnMergeIteratorTest, KeepsByteOrderWhenMergingMaterializedSubcolumn) { |
| 830 | + auto sparse = variant_util::create_variant_binary_column(); |
| 831 | + auto& map = assert_cast<ColumnMap&>(*sparse); |
| 832 | + DataTypeInt64 int_type; |
| 833 | + auto ints = ColumnInt64::create(); |
| 834 | + ints->insert_value(1); |
| 835 | + ints->insert_value(2); |
| 836 | + append_storage_cell(map, "a-b", int_type, *ints, 0); |
| 837 | + append_storage_cell(map, "a.c", int_type, *ints, 1); |
| 838 | + map.get_offsets().push_back(2); |
| 839 | + |
| 840 | + auto counters = std::make_shared<SparseReadCounters>(); |
| 841 | + auto cache = std::make_shared<BinaryColumnCache>( |
| 842 | + std::make_unique<FixedSparseIterator>(std::move(sparse), counters), |
| 843 | + variant_util::create_variant_binary_column()); |
| 844 | + |
| 845 | + // Materialized path a.d; the dense iterator yields one nullable bigint row: 3. |
| 846 | + auto dense_type = std::make_shared<DataTypeNullable>(std::make_shared<DataTypeInt64>()); |
| 847 | + auto dense_values = ColumnNullable::create(ColumnInt64::create(), ColumnUInt8::create()); |
| 848 | + assert_cast<ColumnInt64&>(dense_values->get_nested_column()).insert_value(3); |
| 849 | + dense_values->get_null_map_data().push_back(0); |
| 850 | + SubstreamReaderTree src_subcolumns; |
| 851 | + ASSERT_TRUE(src_subcolumns.add( |
| 852 | + PathInData("a.d"), |
| 853 | + SubstreamIterator(ColumnNullable::create(ColumnInt64::create(), ColumnUInt8::create()), |
| 854 | + std::make_unique<FixedSparseIterator>( |
| 855 | + std::move(dense_values), |
| 856 | + std::make_shared<SparseReadCounters>()), |
| 857 | + dense_type))); |
| 858 | + |
| 859 | + VariantCompactionPaths path_set_info; |
| 860 | + OlapReaderStatistics stats; |
| 861 | + StorageReadOptions read_options(stats); |
| 862 | + SparseColumnMergeIterator iterator(path_set_info, std::move(cache), std::move(src_subcolumns), |
| 863 | + &read_options); |
| 864 | + ColumnIteratorOptions iterator_options; |
| 865 | + iterator_options.stats = &stats; |
| 866 | + ASSERT_TRUE(iterator.init(iterator_options).ok()); |
| 867 | + ASSERT_TRUE(iterator.seek_to_ordinal(0).ok()); |
| 868 | + auto dst = variant_util::create_variant_binary_column(); |
| 869 | + size_t rows = 1; |
| 870 | + bool has_null = false; |
| 871 | + ASSERT_TRUE(iterator.next_batch(&rows, dst, &has_null).ok()); |
| 872 | + ASSERT_EQ(dst->size(), 1); |
| 873 | + const auto& merged = assert_cast<const ColumnMap&>(*dst); |
| 874 | + const auto& paths = assert_cast<const ColumnString&>(merged.get_keys()); |
| 875 | + std::vector<std::string> actual; |
| 876 | + const size_t begin = merged.offset_at(0); |
| 877 | + const size_t end = begin + merged.size_at(0); |
| 878 | + for (size_t cell = begin; cell < end; ++cell) { |
| 879 | + actual.push_back(paths.get_data_at(cell).to_string()); |
| 880 | + } |
| 881 | + EXPECT_EQ(actual, (std::vector<std::string> {"a-b", "a.c", "a.d"})); |
| 882 | +} |
| 883 | + |
792 | 884 | } // namespace |
793 | 885 | } // namespace doris::segment_v2 |
0 commit comments