@@ -130,14 +130,17 @@ class OrderedDataCompactionTest : public ::testing::Test {
130130 int32_t _saved_ordered_data_compaction_min_segment_size = 0 ;
131131 int32_t _saved_segments_key_bounds_truncation_threshold = 0 ;
132132
133- TabletSchemaSPtr create_schema (KeysType keys_type = DUP_KEYS ) {
133+ TabletSchemaSPtr create_schema (KeysType keys_type = DUP_KEYS , bool with_version_column = false ,
134+ bool with_commit_tso_column = false ,
135+ bool unset_version_col_idx = false ) {
134136 TabletSchemaSPtr tablet_schema = std::make_shared<TabletSchema>();
135137 TabletSchemaPB tablet_schema_pb;
136138 tablet_schema_pb.set_keys_type (keys_type);
137139 tablet_schema_pb.set_num_short_key_columns (1 );
138140 tablet_schema_pb.set_num_rows_per_row_block (1024 );
139141 tablet_schema_pb.set_compress_kind (COMPRESS_NONE );
140- tablet_schema_pb.set_next_column_unique_id (4 );
142+ tablet_schema_pb.set_next_column_unique_id (4 + with_version_column +
143+ with_commit_tso_column);
141144
142145 ColumnPB* column_1 = tablet_schema_pb.add_column ();
143146 column_1->set_unique_id (1 );
@@ -174,6 +177,38 @@ class OrderedDataCompactionTest : public ::testing::Test {
174177 column_3->set_is_bf_column (false );
175178 }
176179
180+ if (with_version_column) {
181+ ColumnPB* version_column = tablet_schema_pb.add_column ();
182+ version_column->set_unique_id (4 );
183+ version_column->set_name (VERSION_COL );
184+ version_column->set_type (" BIGINT" );
185+ version_column->set_is_key (false );
186+ version_column->set_is_nullable (false );
187+ version_column->set_visible (false );
188+ version_column->set_default_value (" 0" );
189+ version_column->set_length (8 );
190+ version_column->set_index_length (8 );
191+ version_column->set_aggregation (" REPLACE" );
192+ if (!unset_version_col_idx) {
193+ tablet_schema_pb.set_version_col_idx (3 );
194+ }
195+ }
196+
197+ if (with_commit_tso_column) {
198+ tablet_schema_pb.set_commit_tso_col_idx (tablet_schema_pb.column_size ());
199+ ColumnPB* tso_column = tablet_schema_pb.add_column ();
200+ tso_column->set_unique_id (tablet_schema_pb.column_size ());
201+ tso_column->set_name (COMMIT_TSO_COL );
202+ tso_column->set_type (" BIGINT" );
203+ tso_column->set_is_key (false );
204+ tso_column->set_is_nullable (false );
205+ tso_column->set_visible (false );
206+ tso_column->set_default_value (" 0" );
207+ tso_column->set_length (8 );
208+ tso_column->set_index_length (8 );
209+ tso_column->set_aggregation (" NONE" );
210+ }
211+
177212 tablet_schema->init_from_pb (tablet_schema_pb);
178213 return tablet_schema;
179214 }
@@ -301,7 +336,8 @@ class OrderedDataCompactionTest : public ::testing::Test {
301336 RowsetSharedPtr create_rowset (
302337 TabletSchemaSPtr tablet_schema, TabletSharedPtr tablet,
303338 const SegmentsOverlapPB& overlap,
304- std::vector<std::vector<std::tuple<int64_t , int64_t >>> rowset_data) {
339+ std::vector<std::vector<std::tuple<int64_t , int64_t >>> rowset_data,
340+ Version version = Version(-1 , -1 ), int64_t stored_version = 0) {
305341 RowsetWriterContext writer_context;
306342 if (overlap == NONOVERLAPPING ) {
307343 for (auto i = 1 ; i < rowset_data.size (); i++) {
@@ -314,6 +350,9 @@ class OrderedDataCompactionTest : public ::testing::Test {
314350 }
315351 create_rowset_writer_context (tablet_schema, tablet->tablet_path (), overlap, UINT32_MAX ,
316352 &writer_context);
353+ if (version.first >= 0 ) {
354+ writer_context.version = version;
355+ }
317356
318357 auto res = RowsetFactory::create_rowset_writer (*engine_ref, writer_context, true );
319358 EXPECT_TRUE (res.has_value ()) << res.error ();
@@ -333,6 +372,15 @@ class OrderedDataCompactionTest : public ::testing::Test {
333372 uint8_t num = 0 ;
334373 columns[2 ]->insert_data ((const char *)&num, sizeof (num));
335374 }
375+ if (auto version_idx = tablet_schema->field_index (VERSION_COL ); version_idx >= 0 ) {
376+ columns[version_idx]->insert_data ((const char *)&stored_version,
377+ sizeof (stored_version));
378+ }
379+ if (tablet_schema->commit_tso_col_idx () >= 0 ) {
380+ int64_t tso_placeholder = 0 ;
381+ columns[tablet_schema->commit_tso_col_idx ()]->insert_data (
382+ (const char *)&tso_placeholder, sizeof (tso_placeholder));
383+ }
336384 num_rows++;
337385 }
338386 block.set_columns (std::move (columns));
@@ -349,6 +397,32 @@ class OrderedDataCompactionTest : public ::testing::Test {
349397 return rowset;
350398 }
351399
400+ std::vector<int64_t > read_hidden_values (Rowset* rowset, const TabletSchemaSPtr& tablet_schema,
401+ ColumnId hidden_column_id) {
402+ auto read_schema = std::make_shared<ReadSchema>(project_columns_by_ordinal (
403+ tablet_schema->columns (), std::vector<ColumnId> {0 , hidden_column_id}));
404+ EXPECT_TRUE (read_schema->init_from_tablet_schema (*tablet_schema, false , false ).ok ());
405+ RowsetReaderContext reader_context;
406+ reader_context.read_schema = read_schema;
407+ RowsetReaderSharedPtr reader;
408+ create_and_init_rowset_reader (rowset, reader_context, &reader);
409+ std::vector<int64_t > values;
410+ Status status = Status::OK ();
411+ do {
412+ Block block = read_schema->create_read_block ();
413+ status = reader->next_batch (&block);
414+ for (size_t i = 0 ; i < block.rows (); ++i) {
415+ values.push_back (block.get_by_position (1 ).column ->get_int (i));
416+ }
417+ } while (status.ok ());
418+ EXPECT_EQ (status, Status::Error<END_OF_FILE >(" " ));
419+ return values;
420+ }
421+
422+ std::vector<int64_t > read_versions (Rowset* rowset, const TabletSchemaSPtr& tablet_schema) {
423+ return read_hidden_values (rowset, tablet_schema, tablet_schema->field_index (VERSION_COL ));
424+ }
425+
352426 void init_rs_meta (RowsetMetaSharedPtr& pb1, int64_t start, int64_t end) {
353427 std::string json_rowset_meta = R"( {
354428 "rowset_id": 540085,
@@ -559,6 +633,129 @@ TEST_F(OrderedDataCompactionTest, test_01) {
559633 }
560634}
561635
636+ TEST_F (OrderedDataCompactionTest, test_version_col_singleton_rowset_not_linked) {
637+ auto tablet_schema = create_schema (UNIQUE_KEYS , true );
638+ auto tablet = create_tablet (*tablet_schema, false , 10000 , false );
639+ ASSERT_TRUE (io::global_local_filesystem ()->create_directory (tablet->tablet_path ()).ok ());
640+
641+ std::vector<std::vector<std::vector<std::tuple<int64_t , int64_t >>>> input_data;
642+ generate_input_data (2 , 1 , 2 , input_data);
643+ std::vector<RowsetSharedPtr> input_rowsets;
644+ for (const auto & rowset_data : input_data) {
645+ auto rowset = create_rowset (tablet_schema, tablet, NONOVERLAPPING , rowset_data);
646+ ASSERT_EQ (rowset->start_version (), rowset->end_version ());
647+ ASSERT_EQ (rowset->num_rows (), 2 );
648+ EXPECT_EQ (read_versions (rowset.get (), tablet_schema),
649+ (std::vector<int64_t > {rowset->end_version (), rowset->end_version ()}));
650+ input_rowsets.push_back (std::move (rowset));
651+ }
652+
653+ CumulativeCompaction compaction (*engine_ref, tablet);
654+ compaction._input_rowsets = std::move (input_rowsets);
655+ ASSERT_FALSE (compaction.handle_ordered_data_compaction ());
656+ }
657+
658+ TEST_F (OrderedDataCompactionTest, test_version_col_with_unset_index_singleton_not_linked) {
659+ // A schema created without light schema change has the VERSION column but no index in PB.
660+ auto tablet_schema = create_schema (UNIQUE_KEYS , true , false , true );
661+ ASSERT_EQ (tablet_schema->version_col_idx (), -1 );
662+ ASSERT_EQ (tablet_schema->field_index (VERSION_COL ), 3 );
663+ auto tablet = create_tablet (*tablet_schema, false , 10000 , false );
664+ ASSERT_TRUE (io::global_local_filesystem ()->create_directory (tablet->tablet_path ()).ok ());
665+
666+ std::vector<std::vector<std::vector<std::tuple<int64_t , int64_t >>>> input_data;
667+ generate_input_data (2 , 1 , 2 , input_data);
668+ std::vector<RowsetSharedPtr> input_rowsets;
669+ for (const auto & rowset_data : input_data) {
670+ auto rowset = create_rowset (tablet_schema, tablet, NONOVERLAPPING , rowset_data);
671+ ASSERT_EQ (rowset->start_version (), rowset->end_version ());
672+ ASSERT_EQ (rowset->num_rows (), 2 );
673+ EXPECT_EQ (read_versions (rowset.get (), tablet_schema),
674+ (std::vector<int64_t > {rowset->end_version (), rowset->end_version ()}));
675+ input_rowsets.push_back (std::move (rowset));
676+ }
677+
678+ CumulativeCompaction compaction (*engine_ref, tablet);
679+ compaction._input_rowsets = std::move (input_rowsets);
680+ ASSERT_FALSE (compaction.handle_ordered_data_compaction ());
681+ }
682+
683+ TEST_F (OrderedDataCompactionTest, test_version_col_materialized_range_rowsets_can_link) {
684+ auto tablet_schema = create_schema (UNIQUE_KEYS , true );
685+ auto tablet = create_tablet (*tablet_schema, false , 10000 , false );
686+ ASSERT_TRUE (io::global_local_filesystem ()->create_directory (tablet->tablet_path ()).ok ());
687+
688+ std::vector<std::vector<std::vector<std::tuple<int64_t , int64_t >>>> input_data;
689+ generate_input_data (2 , 1 , 2 , input_data);
690+ auto first = create_rowset (tablet_schema, tablet, NONOVERLAPPING , input_data[0 ],
691+ Version (1000 , 1001 ), 1001 );
692+ auto second = create_rowset (tablet_schema, tablet, NONOVERLAPPING , input_data[1 ],
693+ Version (1002 , 1003 ), 1003 );
694+ EXPECT_EQ (read_versions (first.get (), tablet_schema), (std::vector<int64_t > {1001 , 1001 }));
695+ EXPECT_EQ (read_versions (second.get (), tablet_schema), (std::vector<int64_t > {1003 , 1003 }));
696+
697+ CumulativeCompaction compaction (*engine_ref, tablet);
698+ compaction._input_rowsets = {first, second};
699+ ASSERT_TRUE (compaction.handle_ordered_data_compaction ());
700+ EXPECT_EQ (compaction._output_rowset ->start_version (), 1000 );
701+ EXPECT_EQ (compaction._output_rowset ->end_version (), 1003 );
702+ EXPECT_EQ (read_versions (compaction._output_rowset .get (), tablet_schema),
703+ (std::vector<int64_t > {1001 , 1001 , 1003 , 1003 }));
704+ }
705+
706+ TEST_F (OrderedDataCompactionTest, test_commit_tso_singleton_rowsets_not_linked) {
707+ auto tablet_schema = create_schema (DUP_KEYS , false , true );
708+ auto tablet = create_tablet (*tablet_schema, false , 10000 , false );
709+ ASSERT_TRUE (io::global_local_filesystem ()->create_directory (tablet->tablet_path ()).ok ());
710+
711+ std::vector<std::vector<std::vector<std::tuple<int64_t , int64_t >>>> input_data;
712+ generate_input_data (2 , 1 , 2 , input_data);
713+ std::vector<RowsetSharedPtr> input_rowsets;
714+ for (size_t i = 0 ; i < input_data.size (); ++i) {
715+ auto rowset = create_rowset (tablet_schema, tablet, NONOVERLAPPING , input_data[i]);
716+ rowset->rowset_meta ()->set_commit_tso (100 + i);
717+ EXPECT_EQ (read_hidden_values (rowset.get (), tablet_schema,
718+ tablet_schema->commit_tso_col_idx ()),
719+ (std::vector<int64_t > {100 + static_cast <int64_t >(i),
720+ 100 + static_cast <int64_t >(i)}));
721+ input_rowsets.push_back (std::move (rowset));
722+ }
723+
724+ CumulativeCompaction compaction (*engine_ref, tablet);
725+ compaction._input_rowsets = std::move (input_rowsets);
726+ ASSERT_FALSE (compaction.handle_ordered_data_compaction ());
727+ }
728+
729+ TEST_F (OrderedDataCompactionTest, test_unsafe_tail_does_not_block_safe_prefix) {
730+ auto tablet_schema = create_schema (UNIQUE_KEYS , true );
731+ auto tablet = create_tablet (*tablet_schema, false , 10000 , false );
732+ ASSERT_TRUE (io::global_local_filesystem ()->create_directory (tablet->tablet_path ()).ok ());
733+
734+ std::vector<std::vector<std::vector<std::tuple<int64_t , int64_t >>>> input_data;
735+ generate_input_data (5 , 1 , 2 , input_data);
736+ std::vector<RowsetSharedPtr> input_rowsets;
737+ for (size_t i = 0 ; i < input_data.size () - 1 ; ++i) {
738+ input_rowsets.push_back (create_rowset (tablet_schema, tablet, NONOVERLAPPING , input_data[i],
739+ Version (1000 + 2 * i, 1001 + 2 * i), 1001 + 2 * i));
740+ }
741+ // Mark a two-segment singleton tail untidy; it must not veto the tidy range prefix.
742+ auto tail_data = std::vector<std::vector<std::tuple<int64_t , int64_t >>> {
743+ {input_data.back ()[0 ][0 ]}, {input_data.back ()[0 ][0 ]}};
744+ auto tail = create_rowset (tablet_schema, tablet, OVERLAPPING , tail_data, Version (1008 , 1008 ));
745+ ASSERT_EQ (tail->num_segments (), 2 );
746+ ASSERT_TRUE (tail->is_segments_overlapping ());
747+ input_rowsets.push_back (tail);
748+
749+ CumulativeCompaction compaction (*engine_ref, tablet);
750+ compaction._input_rowsets = std::move (input_rowsets);
751+ ASSERT_TRUE (compaction.handle_ordered_data_compaction ());
752+ EXPECT_EQ (compaction._input_rowsets .size (), 4 );
753+ EXPECT_EQ (compaction._output_rowset ->start_version (), 1000 );
754+ EXPECT_EQ (compaction._output_rowset ->end_version (), 1007 );
755+ EXPECT_EQ (read_versions (compaction._output_rowset .get (), tablet_schema),
756+ (std::vector<int64_t > {1001 , 1001 , 1003 , 1003 , 1005 , 1005 , 1007 , 1007 }));
757+ }
758+
562759TEST_F (OrderedDataCompactionTest, test_index_disk_size) {
563760 auto num_input_rowset = 3 ;
564761 auto num_segments = 2 ;
0 commit comments