Skip to content

Commit ad7b847

Browse files
committed
[refactor](storage) Clarify compaction row-id column naming
### What problem does this PR solve? Issue Number: None Related PR: apache#69195 Problem Summary: Give internal compaction row-id reads the dedicated COMPACTION_ROWID_COL name instead of reusing the removed legacy query column. Rename the schema helper and its callers, and document how source locations follow filtering and merge to build the output row-id mappings used by delete bitmap conversion and inverted-index compaction. The synthetic column is removed before output segments are written. ### Release note None ### Check List (For Author) - Test: Manual checks - Project formatting and check scripts with clang-format 16 on changed files - build-support/check-build-hygiene.sh - git diff --check and stale-name search - Verify that changes only contain intended renames and comments - No rebuild or unit tests for this naming and comment change - Behavior changed: No - Does this need documentation: No
1 parent 8f0febb commit ad7b847

10 files changed

Lines changed: 27 additions & 21 deletions

File tree

‎be/src/common/consts.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,8 @@ const std::string CSV = "csv";
2525
const std::string CSV_WITH_NAMES = "csv_with_names";
2626
const std::string CSV_WITH_NAMES_AND_TYPES = "csv_with_names_and_types";
2727
const std::string GLOBAL_ROWID_COL = "__DORIS_GLOBAL_ROWID_COL__";
28-
// Internal compaction column carrying the physical source row location.
29-
const std::string ROWID_COL = "__DORIS_ROWID_COL__";
28+
// Synthetic row-id column for compaction reads, containing each source row's physical RowLocation.
29+
const std::string COMPACTION_ROWID_COL = "__DORIS_COMPACTION_ROWID_COL__";
3030
const std::string ROW_STORE_COL = "__DORIS_ROW_STORE_COL__";
3131
const std::string DYNAMIC_COLUMN_NAME = "__DORIS_DYNAMIC_COL__";
3232
const std::string PARTIAL_UPDATE_AUTO_INC_COL = "__PARTIAL_UPDATE_AUTO_INC_COLUMN__";

‎be/src/storage/merger.cpp‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,12 @@ Status set_key_range(TabletReader::ReaderParams* reader_params, const TabletSche
9999
return Status::OK();
100100
}
101101

102+
// Compaction reorders and filters rows, so COMPACTION_ROWID_COL carries the original
103+
// (rowset_id, segment_id, row_id) alongside each surviving row through filtering and merge.
104+
// For example, source (rs1, 3, 7) may become output (rs2, 0, 2). Extract the source locations
105+
// so row-id conversion can map them to the new output positions for delete-bitmap conversion
106+
// and inverted-index compaction. Remove this synthetic column before writing the data;
107+
// it is never persisted in the output segment.
102108
Block prepare_compaction_output_block(const Block& block, int32_t rowid_ordinal,
103109
std::vector<RowLocation>* row_locations) {
104110
DORIS_CHECK_GE(rowid_ordinal, 0);
@@ -185,7 +191,7 @@ Status Merger::vmerge_rowsets(BaseTabletSPtr tablet, ReaderType reader_type,
185191

186192
reader_params.read_schema = std::make_shared<ReadSchema>(cur_tablet_schema.columns());
187193
if (convert_rowids) {
188-
reader_params.read_schema->append_rowid_column();
194+
reader_params.read_schema->append_compaction_rowid_column();
189195
}
190196
RETURN_IF_ERROR(reader.init(reader_params));
191197

@@ -398,7 +404,7 @@ Status Merger::vertical_compact_one_group(
398404
reader_params.read_schema = std::make_shared<ReadSchema>(
399405
project_columns_by_ordinal(tablet_schema.columns(), column_group));
400406
if (convert_rowids) {
401-
reader_params.read_schema->append_rowid_column();
407+
reader_params.read_schema->append_compaction_rowid_column();
402408
}
403409
reader_params.batch_size = batch_size;
404410
RETURN_IF_ERROR(reader.init(reader_params, sample_info));

‎be/src/storage/rowset/segcompaction.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -303,7 +303,7 @@ Status SegcompactionWorker::_do_compact_segments(SegCompactionCandidatesSharedPt
303303
auto schema = std::make_shared<ReadSchema>(
304304
project_columns_by_ordinal(ctx.tablet_schema->columns(), column_ids));
305305
if (is_key && _rowid_conversion != nullptr) {
306-
schema->append_rowid_column();
306+
schema->append_compaction_rowid_column();
307307
}
308308
OlapReaderStatistics reader_stats;
309309
std::unique_ptr<VerticalBlockReader> reader;

‎be/src/storage/schema.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -182,14 +182,14 @@ void ReadSchema::append_dropped_columns(std::vector<TabletColumn> columns) {
182182
}
183183
}
184184

185-
void ReadSchema::append_rowid_column() {
185+
void ReadSchema::append_compaction_rowid_column() {
186186
DORIS_CHECK_EQ(_num_block_columns, _read_columns.size());
187187
DORIS_CHECK_EQ(_rowid_ordinal, -1);
188188
auto column = std::make_shared<TabletColumn>(
189189
FieldAggregationMethod::OLAP_FIELD_AGGREGATION_REPLACE,
190190
FieldType::OLAP_FIELD_TYPE_STRING, false, std::numeric_limits<int32_t>::max(),
191191
sizeof(RowLocation));
192-
column->set_name(BeConsts::ROWID_COL);
192+
column->set_name(BeConsts::COMPACTION_ROWID_COL);
193193
_read_columns.emplace_back(column);
194194
_read_types.emplace_back(column->get_vec_type());
195195
++_num_block_columns;

‎be/src/storage/schema.h‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -85,9 +85,9 @@ class ReadSchema {
8585
// Append private historical delete-predicate columns without extending the Block prefix.
8686
void append_dropped_columns(std::vector<TabletColumn> columns);
8787

88-
// Append a synthetic Block column that carries each source row's physical location through
89-
// compaction filtering and merge.
90-
void append_rowid_column();
88+
// Append COMPACTION_ROWID_COL to the Block prefix before any historical delete columns.
89+
// It carries each source row's physical location through compaction filtering and merge.
90+
void append_compaction_rowid_column();
9191

9292
// Create caller-visible Blocks from the FE-slot prefix.
9393
Block create_read_block() const;
@@ -147,7 +147,7 @@ class ReadSchema {
147147
int32_t delete_sign_ordinal() const { return _delete_sign_ordinal; }
148148
// Sequence column used to choose the winning row during merge.
149149
int32_t sequence_ordinal() const { return _sequence_ordinal; }
150-
// Synthetic row identifier returned by rowid-producing scans.
150+
// Synthetic row identifier returned by queries or carried by compaction reads.
151151
int32_t rowid_ordinal() const { return _rowid_ordinal; }
152152
// Rowset version synthesized for single-version reads.
153153
int32_t version_ordinal() const { return _version_ordinal; }
@@ -204,7 +204,7 @@ class ReadSchema {
204204
_sequence_ordinal = i;
205205
}
206206
if (col.name().starts_with(BeConsts::GLOBAL_ROWID_COL) ||
207-
col.name() == BeConsts::ROWID_COL) {
207+
col.name() == BeConsts::COMPACTION_ROWID_COL) {
208208
_rowid_ordinal = i;
209209
}
210210
if (col.name() == VERSION_COL) {

‎be/src/storage/segment/column_reader.h‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -926,9 +926,8 @@ class ArrayFileColumnIterator final : public ColumnIterator {
926926
Status _seek_by_offsets(ordinal_t ord);
927927
};
928928

929-
// Produces the source RowLocation used by compaction row-id conversion. This is an
930-
// internal storage format and is distinct from GlobalRowLoacationV2, which is the
931-
// query-facing distributed row-id format.
929+
// Reads COMPACTION_ROWID_COL by synthesizing the physical source RowLocation during compaction.
930+
// Its payload is RowLocation, distinct from the query-facing GlobalRowLoacationV2 format.
932931
class CompactionRowIdColumnIterator : public ColumnIterator {
933932
public:
934933
CompactionRowIdColumnIterator(RowsetId rowset_id, uint32_t segment_id)

‎be/src/storage/segment/segment_iterator.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1889,7 +1889,8 @@ Status SegmentIterator::_init_column_iterators() {
18891889
if (cid >= _schema->num_block_columns() && !_column_states[cid].has_delete_pred) {
18901890
continue;
18911891
}
1892-
if (_schema->column(cid)->name() == BeConsts::ROWID_COL) {
1892+
if (_schema->column(cid)->name() == BeConsts::COMPACTION_ROWID_COL) {
1893+
// Compaction reads synthesize source locations rather than reading a stored column.
18931894
_column_iterators[cid].reset(
18941895
new CompactionRowIdColumnIterator(_opts.rowset_id, _segment->id()));
18951896
continue;

‎be/test/exec/scan/vgeneric_iterators_test.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -602,7 +602,7 @@ class SeqColumnUtIterator : public RowwiseIterator {
602602
TEST(VGenericIteratorsTest, MergeWithSeqColumn) {
603603
EXPECT_TRUE(1);
604604
auto schema = create_schema();
605-
schema.append_rowid_column();
605+
schema.append_compaction_rowid_column();
606606
auto output_schema = std::make_shared<ReadSchema>(schema);
607607
std::vector<RowwiseIteratorUPtr> inputs;
608608

‎be/test/storage/read_schema_test.cpp‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -462,12 +462,12 @@ TEST(ReadSchemaTest, TabletColumnFromSlotKeepsTypeDetails) {
462462
EXPECT_TRUE(variant_column.variant_enable_doc_mode());
463463
}
464464

465-
TEST(ReadSchemaTest, AppendedRowIdExtendsReadBlockBeforeDroppedColumns) {
465+
TEST(ReadSchemaTest, AppendedCompactionRowIdExtendsReadBlockBeforeDroppedColumns) {
466466
std::vector<TabletColumnPtr> read_columns {create_int_column(10, "k", true)};
467467
auto dropped_column = create_int_column(11, "dropped");
468468
ReadSchema read_schema(std::move(read_columns));
469469

470-
read_schema.append_rowid_column();
470+
read_schema.append_compaction_rowid_column();
471471
read_schema.append_dropped_columns({*dropped_column});
472472

473473
EXPECT_EQ(1, read_schema.rowid_ordinal());
@@ -476,7 +476,7 @@ TEST(ReadSchemaTest, AppendedRowIdExtendsReadBlockBeforeDroppedColumns) {
476476
EXPECT_EQ(2, read_schema.ordinal_by_uid(11));
477477
Block block = read_schema.create_read_block();
478478
ASSERT_EQ(2, block.columns());
479-
EXPECT_EQ(BeConsts::ROWID_COL, block.get_by_position(1).name);
479+
EXPECT_EQ(BeConsts::COMPACTION_ROWID_COL, block.get_by_position(1).name);
480480
EXPECT_EQ(PrimitiveType::TYPE_STRING, block.get_by_position(1).type->get_primitive_type());
481481
}
482482

‎be/test/storage/segment/segment_iterator_count_emit_shortcut_test.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,7 @@ struct Fixture {
113113
tablet_schema = make_tablet_schema();
114114
read_schema = make_read_schema(tablet_schema);
115115
if (with_rowid) {
116-
read_schema->append_rowid_column();
116+
read_schema->append_compaction_rowid_column();
117117
}
118118
StorageReadOptions opts(stats);
119119
iter = std::make_unique<SegmentIterator>(std::make_shared<MockSegment>(tablet_schema),

0 commit comments

Comments
 (0)