Skip to content

Commit a35707d

Browse files
authored
[fix](variant) Write Variant V2 sparse keys in byte order (apache#70001)
1 parent c7d1df4 commit a35707d

4 files changed

Lines changed: 290 additions & 0 deletions

File tree

‎be/src/storage/segment/variant/v2/variant_shredder.cpp‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -509,6 +509,11 @@ struct VariantShredder::Impl {
509509

510510
DorisVector<SparsePlan> sparse_plan = build_sparse_plan(selection, candidate_builders);
511511
select_sparse_statistics(&sparse_plan);
512+
// Readers search full path bytes: a-b must precede a.c, unlike component order.
513+
// Select statistics first to retain the existing encounter-order budget tie-break.
514+
std::ranges::sort(sparse_plan, [](const SparsePlan& left, const SparsePlan& right) {
515+
return *left.path < *right.path;
516+
});
512517
RETURN_IF_ERROR(append_binary_rows(sparse_plan, sparse_maps));
513518
DorisVector<VariantStatistics> bucket_statistics(options.sparse_bucket_count);
514519
publish_sparse_statistics(sparse_plan, &bucket_statistics, result);

‎be/test/storage/variant/variant_column_writer_reader_test.cpp‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,7 @@
7474
#include "storage/segment/variant/nested_group_provider.h"
7575
#include "storage/segment/variant/nested_group_streaming_write_plan.h"
7676
#include "storage/segment/variant/sparse_column_merge_iterator.h"
77+
#include "storage/segment/variant/v2/variant_assembler.h"
7778
#include "storage/segment/variant/v2/variant_column_writer.h"
7879
#include "storage/segment/variant/v2/variant_path_builder.h"
7980
#include "storage/segment/variant/v2/variant_shredder.h"
@@ -968,6 +969,92 @@ TEST(VariantPathBuilderTest, ShredderReusesCanonicalPathsAcrossAppends) {
968969
EXPECT_EQ(selected.type->to_string(*selected.column, 1), "2");
969970
}
970971

972+
TEST(VariantShredderTest, SparseRowsUseByteOrderWithoutChangingStatisticsSelection) {
973+
ColumnPtr source;
974+
DataTypePtr source_type;
975+
ASSERT_TRUE(create_variant_writer_source(
976+
VariantWriterInput::V2,
977+
{R"({"z":1,"a-b":1,"a b":3,"a$b":4,"a%2Eb":5,"a":{"c":2}})", R"({"z":2})",
978+
R"({"z":3})"},
979+
1, false, {}, &source, &source_type)
980+
.ok());
981+
const auto& values = assert_cast<const ColumnVariantV2&>(*source);
982+
for (const auto layout : {segment_v2::VariantShredderPhysicalLayout::ORDINARY,
983+
segment_v2::VariantShredderPhysicalLayout::DOC}) {
984+
for (const uint32_t buckets : {1, 3}) {
985+
SCOPED_TRACE(testing::Message()
986+
<< "layout=" << static_cast<int>(layout) << " buckets=" << buckets);
987+
segment_v2::VariantShredderOptions options;
988+
options.physical_layout = layout;
989+
options.max_subcolumns_count = 1;
990+
options.sparse_bucket_count = buckets;
991+
options.max_sparse_column_statistics_size = 1;
992+
options.doc_bucket_count = buckets;
993+
options.doc_materialization_min_rows = 4;
994+
segment_v2::VariantShredder shredder(options);
995+
segment_v2::VariantShredder::TestAccess::set_binary_cells_per_chunk(shredder, 2);
996+
ASSERT_TRUE(shredder.append(values.read_view(), 0, values.size()).ok());
997+
segment_v2::VariantShreddedColumns output;
998+
ASSERT_TRUE(shredder.finish(&output).ok());
999+
ASSERT_EQ(output.binary_buckets.size(), buckets);
1000+
for (const auto& bucket : output.binary_buckets) {
1001+
const auto& map = assert_cast<const ColumnMap&>(*bucket.column);
1002+
const auto& keys = assert_cast<const ColumnString&>(map.get_keys());
1003+
for (size_t row = 0; row < map.size(); ++row) {
1004+
std::vector<std::string> actual;
1005+
for (size_t cell = map.offset_at(row);
1006+
cell < map.offset_at(row) + map.size_at(row); ++cell) {
1007+
actual.push_back(keys.get_data_at(cell).to_string());
1008+
}
1009+
auto expected = actual;
1010+
std::ranges::sort(expected);
1011+
EXPECT_EQ(actual, expected) << "row=" << row;
1012+
}
1013+
}
1014+
if (layout == segment_v2::VariantShredderPhysicalLayout::ORDINARY) {
1015+
ASSERT_EQ(output.materialized.size(), 1);
1016+
EXPECT_EQ(output.materialized.front().path.get_path(), "z");
1017+
const std::vector<std::pair<std::string, std::string>> cases {
1018+
{"a.c", "2"}, {"a-b", "1"}, {"a b", "3"}, {"a$b", "4"}, {"a%2Eb", "5"}};
1019+
for (const auto& [path, expected] : cases) {
1020+
SCOPED_TRACE(path);
1021+
const uint32_t bucket = variant_util::variant_binary_shard_of(
1022+
{path.data(), path.size()}, buckets);
1023+
const auto& map =
1024+
assert_cast<const ColumnMap&>(*output.binary_buckets[bucket].column);
1025+
segment_v2::variant_v2::VariantAssemblerOptions read_options;
1026+
read_options.requested_path = PathInData(path);
1027+
read_options.storage_map_kind = segment_v2::variant_v2::StorageMapKind::SPARSE;
1028+
auto assembler = segment_v2::variant_v2::VariantAssembler::create(
1029+
std::move(read_options));
1030+
ASSERT_TRUE(assembler.has_value()) << assembler.error();
1031+
ColumnNullable::MutablePtr result;
1032+
ASSERT_TRUE((*assembler)
1033+
->assemble({.num_rows = values.size(),
1034+
.materialized_columns = {},
1035+
.storage_map = &map},
1036+
&result)
1037+
.ok());
1038+
const auto& read_values =
1039+
assert_cast<const ColumnVariantV2&>(result->get_nested_column());
1040+
EXPECT_EQ(variant_v2_json_at(read_values, 0), expected);
1041+
EXPECT_FALSE(result->is_null_at(0));
1042+
EXPECT_TRUE(result->is_null_at(1));
1043+
EXPECT_TRUE(result->is_null_at(2));
1044+
}
1045+
if (buckets == 1) {
1046+
// Component order encounters a.c first; physical byte order must not
1047+
// change which path wins the one-entry statistics budget.
1048+
EXPECT_EQ(output.statistics.sparse_column_non_null_size.size(), 1);
1049+
EXPECT_EQ(output.statistics.sparse_column_non_null_size.at("a.c"), 1);
1050+
EXPECT_EQ(output.binary_buckets.front().statistics.sparse_column_non_null_size,
1051+
output.statistics.sparse_column_non_null_size);
1052+
}
1053+
}
1054+
}
1055+
}
1056+
}
1057+
9711058
TEST(VariantShredderTest, ReusesRootAndNestedPathsForSharedMetadata) {
9721059
DataTypeVariantV2SerDe serde;
9731060
DataTypeSerDe::FormatOptions format_options;
Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
-- This file is automatically generated. You should know what you did if you want to edit this
2+
-- !before_compaction --
3+
1 11 12 13 14 15 {"c":15} {"a":{"c":15},"a b":12,"a$b":13,"a%2Eb":14,"a-b":11,"z":1}
4+
2 \N \N \N \N \N \N {"z":2}
5+
3 \N \N \N \N \N \N {"z":3}
6+
4 21 22 23 24 25 {"c":25} {"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4}
7+
5 \N \N \N \N \N \N {"z":5}
8+
6 \N \N \N \N \N \N {"z":6}
9+
10+
-- !after_compaction --
11+
1 11 12 13 14 15 {"c":15} {"a":{"c":15},"a b":12,"a$b":13,"a%2Eb":14,"a-b":11,"z":1}
12+
2 \N \N \N \N \N \N {"z":2}
13+
3 \N \N \N \N \N \N {"z":3}
14+
4 21 22 23 24 25 {"c":25} {"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4}
15+
5 \N \N \N \N \N \N {"z":5}
16+
6 \N \N \N \N \N \N {"z":6}
17+
18+
-- !before_compaction --
19+
1 11 12 13 14 15 {"c":15} {"a":{"c":15},"a b":12,"a$b":13,"a%2Eb":14,"a-b":11,"z":1}
20+
2 \N \N \N \N \N \N {"z":2}
21+
3 \N \N \N \N \N \N {"z":3}
22+
4 21 22 23 24 25 {"c":25} {"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4}
23+
5 \N \N \N \N \N \N {"z":5}
24+
6 \N \N \N \N \N \N {"z":6}
25+
26+
-- !after_compaction --
27+
1 11 12 13 14 15 {"c":15} {"a":{"c":15},"a b":12,"a$b":13,"a%2Eb":14,"a-b":11,"z":1}
28+
2 \N \N \N \N \N \N {"z":2}
29+
3 \N \N \N \N \N \N {"z":3}
30+
4 21 22 23 24 25 {"c":25} {"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4}
31+
5 \N \N \N \N \N \N {"z":5}
32+
6 \N \N \N \N \N \N {"z":6}
33+
34+
-- !before_compaction --
35+
1 11 12 13 14 15 {"c":15} {"a":{"c":15},"a b":12,"a$b":13,"a%2Eb":14,"a-b":11,"z":1}
36+
2 \N \N \N \N \N \N {"z":2}
37+
3 \N \N \N \N \N \N {"z":3}
38+
4 21 22 23 24 25 {"c":25} {"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4}
39+
5 \N \N \N \N \N \N {"z":5}
40+
6 \N \N \N \N \N \N {"z":6}
41+
42+
-- !after_compaction --
43+
1 11 12 13 14 15 {"c":15} {"a":{"c":15},"a b":12,"a$b":13,"a%2Eb":14,"a-b":11,"z":1}
44+
2 \N \N \N \N \N \N {"z":2}
45+
3 \N \N \N \N \N \N {"z":3}
46+
4 21 22 23 24 25 {"c":25} {"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4}
47+
5 \N \N \N \N \N \N {"z":5}
48+
6 \N \N \N \N \N \N {"z":6}
49+
50+
-- !before_compaction --
51+
1 11 12 13 14 15 {"c":15} {"a":{"c":15},"a b":12,"a$b":13,"a%2Eb":14,"a-b":11,"z":1}
52+
2 \N \N \N \N \N \N {"z":2}
53+
3 \N \N \N \N \N \N {"z":3}
54+
4 21 22 23 24 25 {"c":25} {"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4}
55+
5 \N \N \N \N \N \N {"z":5}
56+
6 \N \N \N \N \N \N {"z":6}
57+
58+
-- !after_compaction --
59+
1 11 12 13 14 15 {"c":15} {"a":{"c":15},"a b":12,"a$b":13,"a%2Eb":14,"a-b":11,"z":1}
60+
2 \N \N \N \N \N \N {"z":2}
61+
3 \N \N \N \N \N \N {"z":3}
62+
4 21 22 23 24 25 {"c":25} {"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4}
63+
5 \N \N \N \N \N \N {"z":5}
64+
6 \N \N \N \N \N \N {"z":6}
65+
66+
-- !before_compaction --
67+
1 31 32 33 34 35 {"c":35} {"a":{"c":35},"a b":32,"a$b":33,"a%2Eb":34,"a-b":31,"z":1}
68+
2 \N \N \N \N \N \N {"z":2}
69+
3 \N \N \N \N \N \N {"z":3}
70+
4 21 22 23 24 25 {"c":25} {"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4}
71+
5 \N \N \N \N \N \N {"z":5}
72+
6 \N \N \N \N \N \N {"z":6}
73+
74+
-- !after_compaction --
75+
1 31 32 33 34 35 {"c":35} {"a":{"c":35},"a b":32,"a$b":33,"a%2Eb":34,"a-b":31,"z":1}
76+
2 \N \N \N \N \N \N {"z":2}
77+
3 \N \N \N \N \N \N {"z":3}
78+
4 21 22 23 24 25 {"c":25} {"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4}
79+
5 \N \N \N \N \N \N {"z":5}
80+
6 \N \N \N \N \N \N {"z":6}
81+
82+
-- !before_compaction --
83+
1 31 32 33 34 35 {"c":35} {"a":{"c":35},"a b":32,"a$b":33,"a%2Eb":34,"a-b":31,"z":1}
84+
2 \N \N \N \N \N \N {"z":2}
85+
3 \N \N \N \N \N \N {"z":3}
86+
4 21 22 23 24 25 {"c":25} {"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4}
87+
5 \N \N \N \N \N \N {"z":5}
88+
6 \N \N \N \N \N \N {"z":6}
89+
90+
-- !after_compaction --
91+
1 31 32 33 34 35 {"c":35} {"a":{"c":35},"a b":32,"a$b":33,"a%2Eb":34,"a-b":31,"z":1}
92+
2 \N \N \N \N \N \N {"z":2}
93+
3 \N \N \N \N \N \N {"z":3}
94+
4 21 22 23 24 25 {"c":25} {"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4}
95+
5 \N \N \N \N \N \N {"z":5}
96+
6 \N \N \N \N \N \N {"z":6}
97+
98+
-- !before_compaction --
99+
1 31 32 33 34 35 {"c":35} {"a":{"c":35},"a b":32,"a$b":33,"a%2Eb":34,"a-b":31,"z":1}
100+
2 \N \N \N \N \N \N {"z":2}
101+
3 \N \N \N \N \N \N {"z":3}
102+
4 21 22 23 24 25 {"c":25} {"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4}
103+
5 \N \N \N \N \N \N {"z":5}
104+
6 \N \N \N \N \N \N {"z":6}
105+
106+
-- !after_compaction --
107+
1 31 32 33 34 35 {"c":35} {"a":{"c":35},"a b":32,"a$b":33,"a%2Eb":34,"a-b":31,"z":1}
108+
2 \N \N \N \N \N \N {"z":2}
109+
3 \N \N \N \N \N \N {"z":3}
110+
4 21 22 23 24 25 {"c":25} {"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4}
111+
5 \N \N \N \N \N \N {"z":5}
112+
6 \N \N \N \N \N \N {"z":6}
113+
114+
-- !before_compaction --
115+
1 31 32 33 34 35 {"c":35} {"a":{"c":35},"a b":32,"a$b":33,"a%2Eb":34,"a-b":31,"z":1}
116+
2 \N \N \N \N \N \N {"z":2}
117+
3 \N \N \N \N \N \N {"z":3}
118+
4 21 22 23 24 25 {"c":25} {"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4}
119+
5 \N \N \N \N \N \N {"z":5}
120+
6 \N \N \N \N \N \N {"z":6}
121+
122+
-- !after_compaction --
123+
1 31 32 33 34 35 {"c":35} {"a":{"c":35},"a b":32,"a$b":33,"a%2Eb":34,"a-b":31,"z":1}
124+
2 \N \N \N \N \N \N {"z":2}
125+
3 \N \N \N \N \N \N {"z":3}
126+
4 21 22 23 24 25 {"c":25} {"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4}
127+
5 \N \N \N \N \N \N {"z":5}
128+
6 \N \N \N \N \N \N {"z":6}
129+
Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
suite("sparse_key_byte_order", "p0") {
19+
// The parser rejects an explicit doc-mode property beside sparse properties.
20+
sql "SET default_variant_enable_doc_mode = false"
21+
for (def keyModel : ["DUPLICATE", "UNIQUE"]) {
22+
for (def shardCount : [1, 3]) {
23+
for (def docMode : [false, true]) {
24+
def tableName = "sparse_key_byte_order_${keyModel.toLowerCase()}_${shardCount}_${docMode}"
25+
def mowProperty = keyModel == "UNIQUE" ? ', "enable_unique_key_merge_on_write"="true"' : ''
26+
def variantProperties = docMode ?
27+
'"variant_enable_doc_mode"="true", "variant_enable_nested_group"="false", ' +
28+
'"variant_doc_materialization_min_rows"="1000000", ' +
29+
"\"variant_doc_hash_shard_count\"=\"${shardCount}\"" :
30+
'"variant_max_subcolumns_count"="1", "variant_enable_typed_paths_to_sparse"="false", ' +
31+
'"variant_enable_nested_group"="false", ' +
32+
"\"variant_sparse_hash_shard_count\"=\"${shardCount}\""
33+
sql "DROP TABLE IF EXISTS ${tableName}"
34+
sql """
35+
CREATE TABLE ${tableName} (
36+
id INT,
37+
v VARIANT<PROPERTIES(${variantProperties})>
38+
) ${keyModel} KEY(id)
39+
DISTRIBUTED BY HASH(id) BUCKETS 1
40+
PROPERTIES("replication_num"="1", "disable_auto_compaction"="true"${mowProperty})
41+
"""
42+
// z is hotter than every other leaf, leaving the punctuation keys sparse.
43+
sql """INSERT INTO ${tableName} VALUES
44+
(1, parse_to_variant('{"z":1,"a-b":11,"a b":12,"a\$b":13,"a%2Eb":14,"a":{"c":15}}')),
45+
(2, parse_to_variant('{"z":2}')),
46+
(3, parse_to_variant('{"z":3}'))
47+
"""
48+
sql """INSERT INTO ${tableName} VALUES
49+
(4, parse_to_variant('{"z":4,"a-b":21,"a b":22,"a\$b":23,"a%2Eb":24,"a":{"c":25}}')),
50+
(5, parse_to_variant('{"z":5}')),
51+
(6, parse_to_variant('{"z":6}'))
52+
"""
53+
if (keyModel == "UNIQUE") {
54+
sql """INSERT INTO ${tableName} VALUES
55+
(1, parse_to_variant('{"z":1,"a-b":31,"a b":32,"a\$b":33,"a%2Eb":34,"a":{"c":35}}')),
56+
(2, parse_to_variant('{"z":2}')),
57+
(3, parse_to_variant('{"z":3}'))
58+
"""
59+
}
60+
def projection = """SELECT id, v['a-b'], v['a b'], v['a\$b'], v['a%2Eb'],
61+
v['a']['c'], CAST(v['a'] AS STRING), CAST(v AS STRING)
62+
FROM ${tableName} ORDER BY id"""
63+
qt_before_compaction projection
64+
trigger_and_wait_compaction(tableName, "full", 1800)
65+
qt_after_compaction projection
66+
}
67+
}
68+
}
69+
}

0 commit comments

Comments
 (0)