Skip to content

Commit 4571c06

Browse files
authored
[fix](be) Reject unsafe ordered links for hidden columns (apache#69395)
### What problem does this PR solve? Issue Number: None Related PR: None Problem Summary: A non-empty single-version rowset can store physical placeholders for publish-time hidden VERSION and TSO columns. Its reader synthesizes the published values from rowset metadata, but ordered file-link compaction can combine such rowsets into a multi-version output whose reader uses the physical placeholders. This can return incorrect hidden-column values. The same risk applies to the changelog quick-merge link path. Reject a link only when the rowsets actually selected for linking include such a singleton, so the existing physical merge materializes the values. A UNIQUE KEY merge-on-read table with `light_schema_change=false` can retain the VERSION column while `version_col_idx` is unset; look up VERSION by name to cover that supported case. Safe materialized range rowsets and tidy prefixes remain linkable. Scope: This PR changes ordered-link admission, not row-store JSONB generation, point/Row-ID reads, or historical rowsets already containing incorrect physical values. It does not add a rowset metadata certificate or change the protobuf/storage format. ### Release note Prevent ordered file-link compaction from exposing publish-time hidden VERSION/TSO placeholders. ### Check List (For Author) - Test: - ASAN BE+FE build: `BUILD_TYPE=ASAN ./build.sh --be --fe -j32` passed, including FE Checkstyle. - BE unit tests: `BUILD_TYPE=ASAN ./run-be-ut.sh --run --filter='OrderedDataCompactionTest.*' -j32` passed (7/7). - Worktree-isolated SQL regression: `test_ordered_compaction_hidden_version_no_light_schema_change` and `test_ordered_compaction_hidden_version_column` passed (1/1 each) using the local regression runner on private FE/BE ports. The latter suite's COMMIT_TSO/time-travel branch is feature-flag dependent and was not independently confirmed in this run. - `build-support/check-build-hygiene.sh`, `build-support/check-format.sh`, and `git diff --check` passed. - Changed-file `clang-tidy` was attempted but did not complete cleanly: the local toolchain analysis cannot find `stddef.h` from `/usr/include/wchar.h`. No clang-tidy pass is claimed. - Behavior changed: Yes. Unsafe ordered links fall back to ordinary physical merge; safe range rowsets and selected tidy prefixes can still use file linking. - Does this need documentation: No.
1 parent 7679a4f commit 4571c06

6 files changed

Lines changed: 536 additions & 3 deletions

File tree

‎be/src/storage/compaction/compaction.cpp‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -753,6 +753,24 @@ bool CompactionMixin::handle_ordered_data_compaction() {
753753
}
754754
}
755755

756+
// A non-empty singleton stores placeholders for publish-time VERSION and TSO columns.
757+
// Linking [2-2] and [3-3] into [2-3] would expose those placeholders rather than the
758+
// synthesized values. Check only the rowsets actually linked: the tidy scan below can
759+
// drop an unsafe tail [10-10] while retaining a safe range prefix [2-9].
760+
auto links_unmaterialized_hidden_column = [this] {
761+
return _input_rowsets.front()->start_version() != _input_rowsets.back()->end_version() &&
762+
std::ranges::any_of(_input_rowsets, [](const RowsetSharedPtr& rowset) {
763+
const auto& schema = rowset->tablet_schema();
764+
return rowset->num_rows() > 0 &&
765+
rowset->start_version() == rowset->end_version() &&
766+
// A non-light-schema-change tablet can retain VERSION by name while
767+
// version_col_idx is -1; linking its singleton would expose stored 0.
768+
(schema->field_index(VERSION_COL) >= 0 ||
769+
schema->commit_tso_col_idx() >= 0 ||
770+
schema->changelog_tso_col_idx() >= 0);
771+
});
772+
};
773+
756774
if (is_changelog_compaction) {
757775
DCHECK(!_input_rowsets.empty()) << "tablet=" << _tablet->tablet_id();
758776
auto compaction_level = _input_rowsets.front()->rowset_meta()->compaction_level();
@@ -763,6 +781,9 @@ bool CompactionMixin::handle_ordered_data_compaction() {
763781
if (!can_quick_merge_changelog) {
764782
return false;
765783
}
784+
if (links_unmaterialized_hidden_column()) {
785+
return false;
786+
}
766787
_trigger_quick_merge_by_changelog = true;
767788

768789
// Changelog quick merge at LMax is a special meta/link compaction path selected by
@@ -791,6 +812,9 @@ bool CompactionMixin::handle_ordered_data_compaction() {
791812
}
792813
}
793814
}
815+
if (links_unmaterialized_hidden_column()) {
816+
return false;
817+
}
794818
}
795819

796820
// most rowset of current compaction is nonoverlapping

‎be/test/storage/compaction/ordered_data_compaction_test.cpp‎

Lines changed: 200 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -130,14 +130,17 @@ class OrderedDataCompactionTest : public ::testing::Test {
130130
int32_t _saved_ordered_data_compaction_min_segment_size = 0;
131131
int32_t _saved_segments_key_bounds_truncation_threshold = 0;
132132

133-
TabletSchemaSPtr create_schema(KeysType keys_type = DUP_KEYS) {
133+
TabletSchemaSPtr create_schema(KeysType keys_type = DUP_KEYS, bool with_version_column = false,
134+
bool with_commit_tso_column = false,
135+
bool unset_version_col_idx = false) {
134136
TabletSchemaSPtr tablet_schema = std::make_shared<TabletSchema>();
135137
TabletSchemaPB tablet_schema_pb;
136138
tablet_schema_pb.set_keys_type(keys_type);
137139
tablet_schema_pb.set_num_short_key_columns(1);
138140
tablet_schema_pb.set_num_rows_per_row_block(1024);
139141
tablet_schema_pb.set_compress_kind(COMPRESS_NONE);
140-
tablet_schema_pb.set_next_column_unique_id(4);
142+
tablet_schema_pb.set_next_column_unique_id(4 + with_version_column +
143+
with_commit_tso_column);
141144

142145
ColumnPB* column_1 = tablet_schema_pb.add_column();
143146
column_1->set_unique_id(1);
@@ -174,6 +177,38 @@ class OrderedDataCompactionTest : public ::testing::Test {
174177
column_3->set_is_bf_column(false);
175178
}
176179

180+
if (with_version_column) {
181+
ColumnPB* version_column = tablet_schema_pb.add_column();
182+
version_column->set_unique_id(4);
183+
version_column->set_name(VERSION_COL);
184+
version_column->set_type("BIGINT");
185+
version_column->set_is_key(false);
186+
version_column->set_is_nullable(false);
187+
version_column->set_visible(false);
188+
version_column->set_default_value("0");
189+
version_column->set_length(8);
190+
version_column->set_index_length(8);
191+
version_column->set_aggregation("REPLACE");
192+
if (!unset_version_col_idx) {
193+
tablet_schema_pb.set_version_col_idx(3);
194+
}
195+
}
196+
197+
if (with_commit_tso_column) {
198+
tablet_schema_pb.set_commit_tso_col_idx(tablet_schema_pb.column_size());
199+
ColumnPB* tso_column = tablet_schema_pb.add_column();
200+
tso_column->set_unique_id(tablet_schema_pb.column_size());
201+
tso_column->set_name(COMMIT_TSO_COL);
202+
tso_column->set_type("BIGINT");
203+
tso_column->set_is_key(false);
204+
tso_column->set_is_nullable(false);
205+
tso_column->set_visible(false);
206+
tso_column->set_default_value("0");
207+
tso_column->set_length(8);
208+
tso_column->set_index_length(8);
209+
tso_column->set_aggregation("NONE");
210+
}
211+
177212
tablet_schema->init_from_pb(tablet_schema_pb);
178213
return tablet_schema;
179214
}
@@ -301,7 +336,8 @@ class OrderedDataCompactionTest : public ::testing::Test {
301336
RowsetSharedPtr create_rowset(
302337
TabletSchemaSPtr tablet_schema, TabletSharedPtr tablet,
303338
const SegmentsOverlapPB& overlap,
304-
std::vector<std::vector<std::tuple<int64_t, int64_t>>> rowset_data) {
339+
std::vector<std::vector<std::tuple<int64_t, int64_t>>> rowset_data,
340+
Version version = Version(-1, -1), int64_t stored_version = 0) {
305341
RowsetWriterContext writer_context;
306342
if (overlap == NONOVERLAPPING) {
307343
for (auto i = 1; i < rowset_data.size(); i++) {
@@ -314,6 +350,9 @@ class OrderedDataCompactionTest : public ::testing::Test {
314350
}
315351
create_rowset_writer_context(tablet_schema, tablet->tablet_path(), overlap, UINT32_MAX,
316352
&writer_context);
353+
if (version.first >= 0) {
354+
writer_context.version = version;
355+
}
317356

318357
auto res = RowsetFactory::create_rowset_writer(*engine_ref, writer_context, true);
319358
EXPECT_TRUE(res.has_value()) << res.error();
@@ -333,6 +372,15 @@ class OrderedDataCompactionTest : public ::testing::Test {
333372
uint8_t num = 0;
334373
columns[2]->insert_data((const char*)&num, sizeof(num));
335374
}
375+
if (auto version_idx = tablet_schema->field_index(VERSION_COL); version_idx >= 0) {
376+
columns[version_idx]->insert_data((const char*)&stored_version,
377+
sizeof(stored_version));
378+
}
379+
if (tablet_schema->commit_tso_col_idx() >= 0) {
380+
int64_t tso_placeholder = 0;
381+
columns[tablet_schema->commit_tso_col_idx()]->insert_data(
382+
(const char*)&tso_placeholder, sizeof(tso_placeholder));
383+
}
336384
num_rows++;
337385
}
338386
block.set_columns(std::move(columns));
@@ -349,6 +397,32 @@ class OrderedDataCompactionTest : public ::testing::Test {
349397
return rowset;
350398
}
351399

400+
std::vector<int64_t> read_hidden_values(Rowset* rowset, const TabletSchemaSPtr& tablet_schema,
401+
ColumnId hidden_column_id) {
402+
auto read_schema = std::make_shared<ReadSchema>(project_columns_by_ordinal(
403+
tablet_schema->columns(), std::vector<ColumnId> {0, hidden_column_id}));
404+
EXPECT_TRUE(read_schema->init_from_tablet_schema(*tablet_schema, false, false).ok());
405+
RowsetReaderContext reader_context;
406+
reader_context.read_schema = read_schema;
407+
RowsetReaderSharedPtr reader;
408+
create_and_init_rowset_reader(rowset, reader_context, &reader);
409+
std::vector<int64_t> values;
410+
Status status = Status::OK();
411+
do {
412+
Block block = read_schema->create_read_block();
413+
status = reader->next_batch(&block);
414+
for (size_t i = 0; i < block.rows(); ++i) {
415+
values.push_back(block.get_by_position(1).column->get_int(i));
416+
}
417+
} while (status.ok());
418+
EXPECT_EQ(status, Status::Error<END_OF_FILE>(""));
419+
return values;
420+
}
421+
422+
std::vector<int64_t> read_versions(Rowset* rowset, const TabletSchemaSPtr& tablet_schema) {
423+
return read_hidden_values(rowset, tablet_schema, tablet_schema->field_index(VERSION_COL));
424+
}
425+
352426
void init_rs_meta(RowsetMetaSharedPtr& pb1, int64_t start, int64_t end) {
353427
std::string json_rowset_meta = R"({
354428
"rowset_id": 540085,
@@ -559,6 +633,129 @@ TEST_F(OrderedDataCompactionTest, test_01) {
559633
}
560634
}
561635

636+
TEST_F(OrderedDataCompactionTest, test_version_col_singleton_rowset_not_linked) {
637+
auto tablet_schema = create_schema(UNIQUE_KEYS, true);
638+
auto tablet = create_tablet(*tablet_schema, false, 10000, false);
639+
ASSERT_TRUE(io::global_local_filesystem()->create_directory(tablet->tablet_path()).ok());
640+
641+
std::vector<std::vector<std::vector<std::tuple<int64_t, int64_t>>>> input_data;
642+
generate_input_data(2, 1, 2, input_data);
643+
std::vector<RowsetSharedPtr> input_rowsets;
644+
for (const auto& rowset_data : input_data) {
645+
auto rowset = create_rowset(tablet_schema, tablet, NONOVERLAPPING, rowset_data);
646+
ASSERT_EQ(rowset->start_version(), rowset->end_version());
647+
ASSERT_EQ(rowset->num_rows(), 2);
648+
EXPECT_EQ(read_versions(rowset.get(), tablet_schema),
649+
(std::vector<int64_t> {rowset->end_version(), rowset->end_version()}));
650+
input_rowsets.push_back(std::move(rowset));
651+
}
652+
653+
CumulativeCompaction compaction(*engine_ref, tablet);
654+
compaction._input_rowsets = std::move(input_rowsets);
655+
ASSERT_FALSE(compaction.handle_ordered_data_compaction());
656+
}
657+
658+
TEST_F(OrderedDataCompactionTest, test_version_col_with_unset_index_singleton_not_linked) {
659+
// A schema created without light schema change has the VERSION column but no index in PB.
660+
auto tablet_schema = create_schema(UNIQUE_KEYS, true, false, true);
661+
ASSERT_EQ(tablet_schema->version_col_idx(), -1);
662+
ASSERT_EQ(tablet_schema->field_index(VERSION_COL), 3);
663+
auto tablet = create_tablet(*tablet_schema, false, 10000, false);
664+
ASSERT_TRUE(io::global_local_filesystem()->create_directory(tablet->tablet_path()).ok());
665+
666+
std::vector<std::vector<std::vector<std::tuple<int64_t, int64_t>>>> input_data;
667+
generate_input_data(2, 1, 2, input_data);
668+
std::vector<RowsetSharedPtr> input_rowsets;
669+
for (const auto& rowset_data : input_data) {
670+
auto rowset = create_rowset(tablet_schema, tablet, NONOVERLAPPING, rowset_data);
671+
ASSERT_EQ(rowset->start_version(), rowset->end_version());
672+
ASSERT_EQ(rowset->num_rows(), 2);
673+
EXPECT_EQ(read_versions(rowset.get(), tablet_schema),
674+
(std::vector<int64_t> {rowset->end_version(), rowset->end_version()}));
675+
input_rowsets.push_back(std::move(rowset));
676+
}
677+
678+
CumulativeCompaction compaction(*engine_ref, tablet);
679+
compaction._input_rowsets = std::move(input_rowsets);
680+
ASSERT_FALSE(compaction.handle_ordered_data_compaction());
681+
}
682+
683+
TEST_F(OrderedDataCompactionTest, test_version_col_materialized_range_rowsets_can_link) {
684+
auto tablet_schema = create_schema(UNIQUE_KEYS, true);
685+
auto tablet = create_tablet(*tablet_schema, false, 10000, false);
686+
ASSERT_TRUE(io::global_local_filesystem()->create_directory(tablet->tablet_path()).ok());
687+
688+
std::vector<std::vector<std::vector<std::tuple<int64_t, int64_t>>>> input_data;
689+
generate_input_data(2, 1, 2, input_data);
690+
auto first = create_rowset(tablet_schema, tablet, NONOVERLAPPING, input_data[0],
691+
Version(1000, 1001), 1001);
692+
auto second = create_rowset(tablet_schema, tablet, NONOVERLAPPING, input_data[1],
693+
Version(1002, 1003), 1003);
694+
EXPECT_EQ(read_versions(first.get(), tablet_schema), (std::vector<int64_t> {1001, 1001}));
695+
EXPECT_EQ(read_versions(second.get(), tablet_schema), (std::vector<int64_t> {1003, 1003}));
696+
697+
CumulativeCompaction compaction(*engine_ref, tablet);
698+
compaction._input_rowsets = {first, second};
699+
ASSERT_TRUE(compaction.handle_ordered_data_compaction());
700+
EXPECT_EQ(compaction._output_rowset->start_version(), 1000);
701+
EXPECT_EQ(compaction._output_rowset->end_version(), 1003);
702+
EXPECT_EQ(read_versions(compaction._output_rowset.get(), tablet_schema),
703+
(std::vector<int64_t> {1001, 1001, 1003, 1003}));
704+
}
705+
706+
TEST_F(OrderedDataCompactionTest, test_commit_tso_singleton_rowsets_not_linked) {
707+
auto tablet_schema = create_schema(DUP_KEYS, false, true);
708+
auto tablet = create_tablet(*tablet_schema, false, 10000, false);
709+
ASSERT_TRUE(io::global_local_filesystem()->create_directory(tablet->tablet_path()).ok());
710+
711+
std::vector<std::vector<std::vector<std::tuple<int64_t, int64_t>>>> input_data;
712+
generate_input_data(2, 1, 2, input_data);
713+
std::vector<RowsetSharedPtr> input_rowsets;
714+
for (size_t i = 0; i < input_data.size(); ++i) {
715+
auto rowset = create_rowset(tablet_schema, tablet, NONOVERLAPPING, input_data[i]);
716+
rowset->rowset_meta()->set_commit_tso(100 + i);
717+
EXPECT_EQ(read_hidden_values(rowset.get(), tablet_schema,
718+
tablet_schema->commit_tso_col_idx()),
719+
(std::vector<int64_t> {100 + static_cast<int64_t>(i),
720+
100 + static_cast<int64_t>(i)}));
721+
input_rowsets.push_back(std::move(rowset));
722+
}
723+
724+
CumulativeCompaction compaction(*engine_ref, tablet);
725+
compaction._input_rowsets = std::move(input_rowsets);
726+
ASSERT_FALSE(compaction.handle_ordered_data_compaction());
727+
}
728+
729+
TEST_F(OrderedDataCompactionTest, test_unsafe_tail_does_not_block_safe_prefix) {
730+
auto tablet_schema = create_schema(UNIQUE_KEYS, true);
731+
auto tablet = create_tablet(*tablet_schema, false, 10000, false);
732+
ASSERT_TRUE(io::global_local_filesystem()->create_directory(tablet->tablet_path()).ok());
733+
734+
std::vector<std::vector<std::vector<std::tuple<int64_t, int64_t>>>> input_data;
735+
generate_input_data(5, 1, 2, input_data);
736+
std::vector<RowsetSharedPtr> input_rowsets;
737+
for (size_t i = 0; i < input_data.size() - 1; ++i) {
738+
input_rowsets.push_back(create_rowset(tablet_schema, tablet, NONOVERLAPPING, input_data[i],
739+
Version(1000 + 2 * i, 1001 + 2 * i), 1001 + 2 * i));
740+
}
741+
// Mark a two-segment singleton tail untidy; it must not veto the tidy range prefix.
742+
auto tail_data = std::vector<std::vector<std::tuple<int64_t, int64_t>>> {
743+
{input_data.back()[0][0]}, {input_data.back()[0][0]}};
744+
auto tail = create_rowset(tablet_schema, tablet, OVERLAPPING, tail_data, Version(1008, 1008));
745+
ASSERT_EQ(tail->num_segments(), 2);
746+
ASSERT_TRUE(tail->is_segments_overlapping());
747+
input_rowsets.push_back(tail);
748+
749+
CumulativeCompaction compaction(*engine_ref, tablet);
750+
compaction._input_rowsets = std::move(input_rowsets);
751+
ASSERT_TRUE(compaction.handle_ordered_data_compaction());
752+
EXPECT_EQ(compaction._input_rowsets.size(), 4);
753+
EXPECT_EQ(compaction._output_rowset->start_version(), 1000);
754+
EXPECT_EQ(compaction._output_rowset->end_version(), 1007);
755+
EXPECT_EQ(read_versions(compaction._output_rowset.get(), tablet_schema),
756+
(std::vector<int64_t> {1001, 1001, 1003, 1003, 1005, 1005, 1007, 1007}));
757+
}
758+
562759
TEST_F(OrderedDataCompactionTest, test_index_disk_size) {
563760
auto num_input_rowset = 3;
564761
auto num_segments = 2;
Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
-- This file is automatically generated. You should know what you did if you want to edit this
2+
-- !versions --
3+
10 true
4+
11 true
5+
20 true
6+
21 true
7+
8+
-- !min_max --
9+
true true
10+
11+
-- !version_filters --
12+
2 2
13+
14+
-- !commit_tso --
15+
10 true
16+
11 true
17+
20 true
18+
21 true
19+
20+
-- !time_travel --
21+
10 10
22+
11 11
Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
-- This file is automatically generated. You should know what you did if you want to edit this
2+
-- !versions --
3+
10 true
4+
11 true
5+
20 true
6+
21 true
7+
8+
-- !min_max --
9+
true true
10+
11+
-- !version_filters --
12+
2 2

0 commit comments

Comments
 (0)