Skip to content

Commit df6a324

Browse files
authored
[fix](binlog) Support hidden key columns in row binlog (#65076)
### What problem does this PR solve? Issue Number: #65418 Related PR: #63110 Problem Summary: Row binlog schema and the BE row-binlog writer previously treated normal row-binlog columns as visible source columns only. This misses hidden key columns, while hidden non-key internal columns such as sequence/delete/version/skip-bitmap columns should still be excluded from row binlog. Root cause: In `OlapTable.generateTableRowBinlogSchema()` and `SchemaChangeHandler.addColumnRowBinlog()`, FE generated and maintained row-binlog schema from visible columns only. In `RowBinlogSegmentWriter` / `RowBinlogSourceDataWriter`, BE also used visible-column counts to map source columns to row-binlog normal columns. This PR makes row-binlog normal columns follow a simple source-schema prefix contract: visible columns plus hidden key columns are written, and trailing hidden non-key columns are skipped. FE schema generation and add-column schema-change sync now use this contract. BE row-binlog writing uses the same normal-column count for full writes, partial update filtering, key column materialization, and BEFORE value columns. | File | Change Description | |------|-------------------| | `OlapTable.java` | Generate row-binlog schema from full base schema, include hidden key columns, skip hidden non-key columns, and reject visible/key columns after hidden non-key columns. | | `SchemaChangeHandler.java` | Keep hidden key columns when syncing ADD COLUMN changes to row-binlog schema, while skipping hidden non-key columns. | | `row_binlog_segment_writer.*` | Use row-binlog normal column count instead of visible column count, filter partial-update source cids to normal columns, and collect all key columns in the normal prefix. | | FE/BE/regression tests | Cover hidden key schema generation/writing and hidden non-key exclusion. |
1 parent 77cf85a commit df6a324

9 files changed

Lines changed: 490 additions & 66 deletions

File tree

‎be/src/storage/segment/row_binlog_segment_writer.cpp‎

Lines changed: 95 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,9 @@
1717

1818
#include "storage/segment/row_binlog_segment_writer.h"
1919

20+
#include <algorithm>
21+
#include <iterator>
22+
2023
#include "cloud/config.h"
2124
#include "common/cast_set.h"
2225
#include "core/block/column_with_type_and_name.h"
@@ -76,7 +79,7 @@ Status RowBinlogSegmentWriter::init() {
7679
_binlog_op_col_id = static_cast<uint32_t>(op_col_id);
7780
_normal_col_start_id = tso_col_id == 0 ? BINLOG_COLNUM : 0;
7881

79-
uint32_t normal_col_num = cast_set<uint32_t>(source_schema->num_visible_columns());
82+
uint32_t normal_col_num = cast_set<uint32_t>(_source_data_writer->normal_column_count());
8083
_before_col_start_id = _normal_col_start_id + normal_col_num;
8184

8285
if (!_write_before && _tablet_schema->num_columns() > normal_col_num + BINLOG_COLNUM) {
@@ -117,6 +120,7 @@ Status RowBinlogSegmentWriter::append_block(const Block* block, size_t row_pos,
117120
if (UNLIKELY(source_schema == nullptr)) {
118121
return Status::InternalError("binlog<row> writer missing source_tablet_schema");
119122
}
123+
const size_t normal_column_count = _source_data_writer->normal_column_count();
120124

121125
bool is_partial_update = _binlog_opts.source.partial_update_info &&
122126
_binlog_opts.source.partial_update_info->is_partial_update() &&
@@ -125,6 +129,7 @@ Status RowBinlogSegmentWriter::append_block(const Block* block, size_t row_pos,
125129
std::vector<uint32_t> partial_cids =
126130
is_partial_update ? _binlog_opts.source.partial_update_info->update_cids
127131
: std::vector<uint32_t>();
132+
std::vector<uint32_t> row_binlog_partial_cids = partial_cids;
128133
if (is_partial_update) {
129134
if (block->columns() <= source_schema->num_key_columns() ||
130135
block->columns() >= source_schema->num_columns()) {
@@ -135,11 +140,12 @@ Status RowBinlogSegmentWriter::append_block(const Block* block, size_t row_pos,
135140
_tablet_schema->num_columns()));
136141
}
137142

138-
// binlog don't need invisible column
139-
auto erase_invisible_col = std::remove_if(
140-
partial_cids.begin(), partial_cids.end(),
141-
[&](uint32_t cid) { return cid >= source_schema->num_visible_columns(); });
142-
partial_cids.erase(erase_invisible_col, partial_cids.end());
143+
// Partial update lists source cids. Row-binlog writes visible columns and hidden key
144+
// columns, so hidden non-key source columns are skipped.
145+
auto erase_non_normal_col = std::remove_if(
146+
row_binlog_partial_cids.begin(), row_binlog_partial_cids.end(),
147+
[&](uint32_t cid) { return !_source_data_writer->is_normal_column(cid); });
148+
row_binlog_partial_cids.erase(erase_non_normal_col, row_binlog_partial_cids.end());
143149
}
144150

145151
// get delete_sign_column from source block if has
@@ -181,9 +187,9 @@ Status RowBinlogSegmentWriter::append_block(const Block* block, size_t row_pos,
181187
row_pos, num_rows));
182188
}
183189

184-
size_t max_normal_col_id = _normal_col_start_id + source_schema->num_visible_columns();
185-
RETURN_IF_ERROR(_source_data_writer->fill_normal_columns(_column_writers, _normal_col_start_id,
186-
max_normal_col_id, partial_cids));
190+
size_t max_normal_col_id = _normal_col_start_id + normal_column_count;
191+
RETURN_IF_ERROR(_source_data_writer->fill_normal_columns(
192+
_column_writers, _normal_col_start_id, max_normal_col_id, row_binlog_partial_cids));
187193

188194
// We read historical rows only when we really need them:
189195
// 1. partial update: build the full AFTER row.
@@ -203,9 +209,14 @@ Status RowBinlogSegmentWriter::append_block(const Block* block, size_t row_pos,
203209

204210
if (is_partial_update) {
205211
std::vector<uint32_t> row_binlog_missing_column_ids;
206-
_source_data_writer->filter_source_ids(
207-
_binlog_opts.source.partial_update_info->missing_cids,
208-
row_binlog_missing_column_ids);
212+
row_binlog_missing_column_ids.reserve(
213+
_binlog_opts.source.partial_update_info->missing_cids.size());
214+
// Missing cids are source cids too. Only normal columns have AFTER writers.
215+
for (uint32_t cid : _binlog_opts.source.partial_update_info->missing_cids) {
216+
if (_source_data_writer->is_normal_column(cid)) {
217+
row_binlog_missing_column_ids.emplace_back(cid);
218+
}
219+
}
209220

210221
// build AFTER block (fill missing columns in full_block)
211222
RETURN_IF_ERROR(_historical_data_writer->build_after_block(&full_block, row_pos, num_rows));
@@ -215,7 +226,8 @@ Status RowBinlogSegmentWriter::append_block(const Block* block, size_t row_pos,
215226
RETURN_IF_ERROR(after_convertor->set_source_content_with_specifid_columns(
216227
&full_block, row_pos, num_rows, row_binlog_missing_column_ids));
217228
for (auto cid : row_binlog_missing_column_ids) {
218-
auto converted_cid = _normal_col_start_id + cid;
229+
auto converted_cid =
230+
_normal_col_start_id + _source_data_writer->normal_column_ordinal(cid);
219231
auto converted_result = after_convertor->convert_column_data(cid);
220232
if (!converted_result.first.ok()) {
221233
return converted_result.first;
@@ -230,9 +242,9 @@ Status RowBinlogSegmentWriter::append_block(const Block* block, size_t row_pos,
230242
DCHECK(!_tablet_schema->has_sequence_col());
231243
// _converted_key_columns must be resized before fill binlog columns
232244
_converted_key_columns.resize(_tablet_schema->num_key_columns());
233-
for (size_t i = _normal_col_start_id; i < _tablet_schema->num_key_columns(); i++) {
234-
_converted_key_columns[i] = _source_data_writer->get_converted_column(
235-
cast_set<uint32_t>(i - _normal_col_start_id));
245+
const auto& source_key_columns = _source_data_writer->source_key_columns();
246+
for (size_t i = 0; i < source_key_columns.size(); ++i) {
247+
_converted_key_columns[_normal_col_start_id + i] = source_key_columns[i];
236248
}
237249

238250
std::vector<int64_t> no_operators = std::vector<int64_t> {};
@@ -374,7 +386,14 @@ Status RowBinlogSegmentWriter::_fill_before_columns(size_t num_rows) {
374386
if (UNLIKELY(source_schema == nullptr)) {
375387
return Status::InternalError("row binlog writer missing source_tablet_schema");
376388
}
377-
size_t value_column_num = source_schema->num_visible_value_columns();
389+
std::vector<uint32_t> value_cids;
390+
for (uint32_t cid = 0; cid < source_schema->num_columns(); ++cid) {
391+
const auto& column = source_schema->column(cid);
392+
if (column.visible() && !column.is_key()) {
393+
value_cids.emplace_back(cid);
394+
}
395+
}
396+
size_t value_column_num = value_cids.size();
378397
if (value_column_num == 0) {
379398
// No BEFORE columns in row binlog schema.
380399
return Status::OK();
@@ -401,13 +420,6 @@ Status RowBinlogSegmentWriter::_fill_before_columns(size_t num_rows) {
401420
} else {
402421
DCHECK(_historical_data_writer != nullptr);
403422

404-
std::vector<uint32_t> value_cids;
405-
uint32_t value_start = cast_set<uint32_t>(source_schema->num_key_columns());
406-
uint32_t value_end = cast_set<uint32_t>(source_schema->num_visible_columns());
407-
for (uint32_t cid = value_start; cid < value_end; ++cid) {
408-
value_cids.emplace_back(cid);
409-
}
410-
411423
DCHECK_EQ(before_cids.size(), value_cids.size());
412424
RETURN_IF_ERROR(_historical_data_writer->build_before_block(&before_block, value_cids, 0,
413425
num_rows));
@@ -429,50 +441,79 @@ Status RowBinlogSegmentWriter::_fill_before_columns(size_t num_rows) {
429441
return Status::OK();
430442
}
431443

444+
bool RowBinlogSourceDataWriter::is_normal_column(uint32_t source_cid) const {
445+
return std::find(_normal_column_ids.begin(), _normal_column_ids.end(), source_cid) !=
446+
_normal_column_ids.end();
447+
}
448+
449+
size_t RowBinlogSourceDataWriter::normal_column_ordinal(uint32_t source_cid) const {
450+
auto it = std::find(_normal_column_ids.begin(), _normal_column_ids.end(), source_cid);
451+
DCHECK(it != _normal_column_ids.end()) << source_cid;
452+
return cast_set<size_t>(std::distance(_normal_column_ids.begin(), it));
453+
}
454+
432455
Status RowBinlogSourceDataWriter::init() {
433456
_olap_data_convertor = std::make_unique<OlapBlockDataConvertor>();
434-
// _normal_column_ids: the columns which we need to write into binlog from source block
435457
if (UNLIKELY(_opt.source.tablet_schema == nullptr)) {
436458
return Status::InternalError("row binlog writer missing source_tablet_schema");
437459
}
438-
for (uint32_t i = 0; i < _opt.source.tablet_schema->num_visible_columns(); i++) {
439-
_normal_column_ids.emplace_back(i);
460+
461+
const auto& source_schema = _opt.source.tablet_schema;
462+
_normal_column_ids.clear();
463+
for (uint32_t cid = 0; cid < source_schema->num_columns(); ++cid) {
464+
const auto& column = source_schema->column(cid);
465+
if (!column.visible() && !column.is_key()) {
466+
continue;
467+
}
468+
_normal_column_ids.emplace_back(cid);
440469
}
441-
_olap_data_convertor->reserve(_opt.source.tablet_schema->num_columns());
442-
for (size_t cid = 0; cid < _opt.source.tablet_schema->num_columns(); cid++) {
443-
_olap_data_convertor->add_column_data_convertor(_opt.source.tablet_schema->column(cid));
470+
_olap_data_convertor->reserve(source_schema->num_columns());
471+
for (size_t cid = 0; cid < source_schema->num_columns(); cid++) {
472+
_olap_data_convertor->add_column_data_convertor(source_schema->column(cid));
444473
}
445474
return Status::OK();
446475
}
447476

448477
Status RowBinlogSourceDataWriter::prepare_by_source_block(
449478
const Block* block, size_t row_pos, size_t num_rows,
450479
std::vector<uint32_t>& partial_source_cids, Block* full_block) {
451-
_converted_columns.resize(_normal_column_ids.size());
480+
TabletSchemaSPtr tablet_schema = _opt.source.tablet_schema;
481+
// OlapBlockDataConvertor and historical lookup are indexed by source cid, so keep
482+
// converted columns sparse over the source schema instead of row-binlog normal order.
483+
_converted_columns.assign(tablet_schema->num_columns(), nullptr);
452484

453485
// LOG(INFO) << block->dump_data(0, num_rows);
454486

455487
// convert column data from engine format to storage layer format
456488
size_t col_pos_in_block = 0;
457-
TabletSchemaSPtr tablet_schema = _opt.source.tablet_schema;
458-
const auto& including_cids =
459-
partial_source_cids.empty() ? _normal_column_ids : partial_source_cids;
460-
for (auto& cid : including_cids) {
461-
const ColumnWithTypeAndName& col = block->get_by_position(col_pos_in_block++);
489+
const bool is_partial_update = !partial_source_cids.empty();
490+
const size_t column_count =
491+
is_partial_update ? partial_source_cids.size() : _normal_column_ids.size();
492+
for (size_t ordinal = 0; ordinal < column_count; ++ordinal) {
493+
const uint32_t source_cid =
494+
is_partial_update ? partial_source_cids[ordinal] : _normal_column_ids[ordinal];
495+
DCHECK_LT(source_cid, tablet_schema->num_columns());
496+
const ColumnWithTypeAndName& col =
497+
block->get_by_position(is_partial_update ? col_pos_in_block++ : source_cid);
462498

463499
RETURN_IF_ERROR(_olap_data_convertor->set_source_content_with_specifid_column(
464-
col, row_pos, num_rows, cid));
500+
col, row_pos, num_rows, source_cid));
465501
// olap data convertor alway start from id = 0
466-
auto converted_result = _olap_data_convertor->convert_column_data(cid);
502+
auto converted_result = _olap_data_convertor->convert_column_data(source_cid);
467503
if (!converted_result.first.ok()) {
468504
return converted_result.first;
469505
}
470-
_converted_columns[cid] = converted_result.second;
506+
_converted_columns[source_cid] = converted_result.second;
471507

472-
if (cid < tablet_schema->num_key_columns()) {
473-
_key_columns.push_back(converted_result.second);
508+
full_block->replace_by_position(source_cid, col.column);
509+
}
510+
for (uint32_t cid : _normal_column_ids) {
511+
if (!tablet_schema->column(cid).is_key()) {
512+
continue;
474513
}
475-
full_block->replace_by_position(cid, col.column);
514+
DCHECK_LT(cid, _converted_columns.size());
515+
DCHECK(_converted_columns[cid] != nullptr);
516+
_key_columns.push_back(_converted_columns[cid]);
476517
}
477518
_num_rows = num_rows;
478519

@@ -497,15 +538,20 @@ Status RowBinlogSourceDataWriter::fill_normal_columns(
497538
std::vector<uint32_t>& partial_source_cids) {
498539
DCHECK_EQ(end - start, _normal_column_ids.size());
499540

500-
const auto& including_cids =
501-
partial_source_cids.empty() ? _normal_column_ids : partial_source_cids;
502-
for (size_t cid : including_cids) {
503-
DCHECK(column_writers[start + cid]->get_column()->type() ==
541+
const bool is_partial_update = !partial_source_cids.empty();
542+
const size_t column_count =
543+
is_partial_update ? partial_source_cids.size() : _normal_column_ids.size();
544+
for (size_t ordinal = 0; ordinal < column_count; ++ordinal) {
545+
uint32_t cid =
546+
is_partial_update ? partial_source_cids[ordinal] : _normal_column_ids[ordinal];
547+
size_t normal_ordinal = is_partial_update ? normal_column_ordinal(cid) : ordinal;
548+
DCHECK(column_writers[start + normal_ordinal]->get_column()->type() ==
504549
_opt.source.tablet_schema->columns()[cid]->type())
505550
<< cid;
506-
RETURN_IF_ERROR(column_writers[start + cid]->append(_converted_columns[cid]->get_nullmap(),
507-
_converted_columns[cid]->get_data(),
508-
_num_rows));
551+
DCHECK(_converted_columns[cid] != nullptr);
552+
RETURN_IF_ERROR(column_writers[start + normal_ordinal]->append(
553+
_converted_columns[cid]->get_nullmap(), _converted_columns[cid]->get_data(),
554+
_num_rows));
509555
}
510556

511557
return Status::OK();

‎be/src/storage/segment/row_binlog_segment_writer.h‎

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -44,20 +44,18 @@ class RowBinlogSourceDataWriter {
4444

4545
void clear();
4646

47-
IOlapColumnDataAccessor* get_converted_column(uint32_t cid) { return _converted_columns[cid]; }
48-
4947
bool need_before() const { return _opt.write_before; }
5048

5149
const std::vector<IOlapColumnDataAccessor*>& source_key_columns() const { return _key_columns; }
5250
const IOlapColumnDataAccessor* seq_column() const { return _seq_column; }
5351

5452
std::unique_ptr<OlapBlockDataConvertor>& olap_data_convertor() { return _olap_data_convertor; }
5553

56-
void filter_source_ids(std::vector<uint32_t>& full_cids, std::vector<uint32_t>& res_cids) {
57-
res_cids.reserve(full_cids.size());
58-
std::set_intersection(_normal_column_ids.begin(), _normal_column_ids.end(),
59-
full_cids.begin(), full_cids.end(), std::back_inserter(res_cids));
60-
}
54+
size_t normal_column_count() const { return _normal_column_ids.size(); }
55+
56+
bool is_normal_column(uint32_t source_cid) const;
57+
58+
size_t normal_column_ordinal(uint32_t source_cid) const;
6159

6260
private:
6361
const SegmentWriteBinlogOptions& _opt;

0 commit comments

Comments
 (0)