1717
1818#include " storage/segment/row_binlog_segment_writer.h"
1919
20+ #include < algorithm>
21+ #include < iterator>
22+
2023#include " cloud/config.h"
2124#include " common/cast_set.h"
2225#include " core/block/column_with_type_and_name.h"
@@ -76,7 +79,7 @@ Status RowBinlogSegmentWriter::init() {
7679 _binlog_op_col_id = static_cast <uint32_t >(op_col_id);
7780 _normal_col_start_id = tso_col_id == 0 ? BINLOG_COLNUM : 0 ;
7881
79- uint32_t normal_col_num = cast_set<uint32_t >(source_schema-> num_visible_columns ());
82+ uint32_t normal_col_num = cast_set<uint32_t >(_source_data_writer-> normal_column_count ());
8083 _before_col_start_id = _normal_col_start_id + normal_col_num;
8184
8285 if (!_write_before && _tablet_schema->num_columns () > normal_col_num + BINLOG_COLNUM ) {
@@ -117,6 +120,7 @@ Status RowBinlogSegmentWriter::append_block(const Block* block, size_t row_pos,
117120 if (UNLIKELY (source_schema == nullptr )) {
118121 return Status::InternalError (" binlog<row> writer missing source_tablet_schema" );
119122 }
123+ const size_t normal_column_count = _source_data_writer->normal_column_count ();
120124
121125 bool is_partial_update = _binlog_opts.source .partial_update_info &&
122126 _binlog_opts.source .partial_update_info ->is_partial_update () &&
@@ -125,6 +129,7 @@ Status RowBinlogSegmentWriter::append_block(const Block* block, size_t row_pos,
125129 std::vector<uint32_t > partial_cids =
126130 is_partial_update ? _binlog_opts.source .partial_update_info ->update_cids
127131 : std::vector<uint32_t >();
132+ std::vector<uint32_t > row_binlog_partial_cids = partial_cids;
128133 if (is_partial_update) {
129134 if (block->columns () <= source_schema->num_key_columns () ||
130135 block->columns () >= source_schema->num_columns ()) {
@@ -135,11 +140,12 @@ Status RowBinlogSegmentWriter::append_block(const Block* block, size_t row_pos,
135140 _tablet_schema->num_columns ()));
136141 }
137142
138- // binlog don't need invisible column
139- auto erase_invisible_col = std::remove_if (
140- partial_cids.begin (), partial_cids.end (),
141- [&](uint32_t cid) { return cid >= source_schema->num_visible_columns (); });
142- partial_cids.erase (erase_invisible_col, partial_cids.end ());
143+ // Partial update lists source cids. Row-binlog writes visible columns and hidden key
144+ // columns, so hidden non-key source columns are skipped.
145+ auto erase_non_normal_col = std::remove_if (
146+ row_binlog_partial_cids.begin (), row_binlog_partial_cids.end (),
147+ [&](uint32_t cid) { return !_source_data_writer->is_normal_column (cid); });
148+ row_binlog_partial_cids.erase (erase_non_normal_col, row_binlog_partial_cids.end ());
143149 }
144150
145151 // get delete_sign_column from source block if has
@@ -181,9 +187,9 @@ Status RowBinlogSegmentWriter::append_block(const Block* block, size_t row_pos,
181187 row_pos, num_rows));
182188 }
183189
184- size_t max_normal_col_id = _normal_col_start_id + source_schema-> num_visible_columns () ;
185- RETURN_IF_ERROR (_source_data_writer->fill_normal_columns (_column_writers, _normal_col_start_id,
186- max_normal_col_id, partial_cids ));
190+ size_t max_normal_col_id = _normal_col_start_id + normal_column_count ;
191+ RETURN_IF_ERROR (_source_data_writer->fill_normal_columns (
192+ _column_writers, _normal_col_start_id, max_normal_col_id, row_binlog_partial_cids ));
187193
188194 // We read historical rows only when we really need them:
189195 // 1. partial update: build the full AFTER row.
@@ -203,9 +209,14 @@ Status RowBinlogSegmentWriter::append_block(const Block* block, size_t row_pos,
203209
204210 if (is_partial_update) {
205211 std::vector<uint32_t > row_binlog_missing_column_ids;
206- _source_data_writer->filter_source_ids (
207- _binlog_opts.source .partial_update_info ->missing_cids ,
208- row_binlog_missing_column_ids);
212+ row_binlog_missing_column_ids.reserve (
213+ _binlog_opts.source .partial_update_info ->missing_cids .size ());
214+ // Missing cids are source cids too. Only normal columns have AFTER writers.
215+ for (uint32_t cid : _binlog_opts.source .partial_update_info ->missing_cids ) {
216+ if (_source_data_writer->is_normal_column (cid)) {
217+ row_binlog_missing_column_ids.emplace_back (cid);
218+ }
219+ }
209220
210221 // build AFTER block (fill missing columns in full_block)
211222 RETURN_IF_ERROR (_historical_data_writer->build_after_block (&full_block, row_pos, num_rows));
@@ -215,7 +226,8 @@ Status RowBinlogSegmentWriter::append_block(const Block* block, size_t row_pos,
215226 RETURN_IF_ERROR (after_convertor->set_source_content_with_specifid_columns (
216227 &full_block, row_pos, num_rows, row_binlog_missing_column_ids));
217228 for (auto cid : row_binlog_missing_column_ids) {
218- auto converted_cid = _normal_col_start_id + cid;
229+ auto converted_cid =
230+ _normal_col_start_id + _source_data_writer->normal_column_ordinal (cid);
219231 auto converted_result = after_convertor->convert_column_data (cid);
220232 if (!converted_result.first .ok ()) {
221233 return converted_result.first ;
@@ -230,9 +242,9 @@ Status RowBinlogSegmentWriter::append_block(const Block* block, size_t row_pos,
230242 DCHECK (!_tablet_schema->has_sequence_col ());
231243 // _converted_key_columns must be resized before fill binlog columns
232244 _converted_key_columns.resize (_tablet_schema->num_key_columns ());
233- for ( size_t i = _normal_col_start_id; i < _tablet_schema-> num_key_columns (); i++) {
234- _converted_key_columns[i] = _source_data_writer-> get_converted_column (
235- cast_set< uint32_t >(i - _normal_col_start_id)) ;
245+ const auto & source_key_columns = _source_data_writer-> source_key_columns ();
246+ for ( size_t i = 0 ; i < source_key_columns. size (); ++i) {
247+ _converted_key_columns[_normal_col_start_id + i] = source_key_columns[i] ;
236248 }
237249
238250 std::vector<int64_t > no_operators = std::vector<int64_t > {};
@@ -374,7 +386,14 @@ Status RowBinlogSegmentWriter::_fill_before_columns(size_t num_rows) {
374386 if (UNLIKELY (source_schema == nullptr )) {
375387 return Status::InternalError (" row binlog writer missing source_tablet_schema" );
376388 }
377- size_t value_column_num = source_schema->num_visible_value_columns ();
389+ std::vector<uint32_t > value_cids;
390+ for (uint32_t cid = 0 ; cid < source_schema->num_columns (); ++cid) {
391+ const auto & column = source_schema->column (cid);
392+ if (column.visible () && !column.is_key ()) {
393+ value_cids.emplace_back (cid);
394+ }
395+ }
396+ size_t value_column_num = value_cids.size ();
378397 if (value_column_num == 0 ) {
379398 // No BEFORE columns in row binlog schema.
380399 return Status::OK ();
@@ -401,13 +420,6 @@ Status RowBinlogSegmentWriter::_fill_before_columns(size_t num_rows) {
401420 } else {
402421 DCHECK (_historical_data_writer != nullptr );
403422
404- std::vector<uint32_t > value_cids;
405- uint32_t value_start = cast_set<uint32_t >(source_schema->num_key_columns ());
406- uint32_t value_end = cast_set<uint32_t >(source_schema->num_visible_columns ());
407- for (uint32_t cid = value_start; cid < value_end; ++cid) {
408- value_cids.emplace_back (cid);
409- }
410-
411423 DCHECK_EQ (before_cids.size (), value_cids.size ());
412424 RETURN_IF_ERROR (_historical_data_writer->build_before_block (&before_block, value_cids, 0 ,
413425 num_rows));
@@ -429,50 +441,79 @@ Status RowBinlogSegmentWriter::_fill_before_columns(size_t num_rows) {
429441 return Status::OK ();
430442}
431443
444+ bool RowBinlogSourceDataWriter::is_normal_column (uint32_t source_cid) const {
445+ return std::find (_normal_column_ids.begin (), _normal_column_ids.end (), source_cid) !=
446+ _normal_column_ids.end ();
447+ }
448+
449+ size_t RowBinlogSourceDataWriter::normal_column_ordinal (uint32_t source_cid) const {
450+ auto it = std::find (_normal_column_ids.begin (), _normal_column_ids.end (), source_cid);
451+ DCHECK (it != _normal_column_ids.end ()) << source_cid;
452+ return cast_set<size_t >(std::distance (_normal_column_ids.begin (), it));
453+ }
454+
432455Status RowBinlogSourceDataWriter::init () {
433456 _olap_data_convertor = std::make_unique<OlapBlockDataConvertor>();
434- // _normal_column_ids: the columns which we need to write into binlog from source block
435457 if (UNLIKELY (_opt.source .tablet_schema == nullptr )) {
436458 return Status::InternalError (" row binlog writer missing source_tablet_schema" );
437459 }
438- for (uint32_t i = 0 ; i < _opt.source .tablet_schema ->num_visible_columns (); i++) {
439- _normal_column_ids.emplace_back (i);
460+
461+ const auto & source_schema = _opt.source .tablet_schema ;
462+ _normal_column_ids.clear ();
463+ for (uint32_t cid = 0 ; cid < source_schema->num_columns (); ++cid) {
464+ const auto & column = source_schema->column (cid);
465+ if (!column.visible () && !column.is_key ()) {
466+ continue ;
467+ }
468+ _normal_column_ids.emplace_back (cid);
440469 }
441- _olap_data_convertor->reserve (_opt. source . tablet_schema ->num_columns ());
442- for (size_t cid = 0 ; cid < _opt. source . tablet_schema ->num_columns (); cid++) {
443- _olap_data_convertor->add_column_data_convertor (_opt. source . tablet_schema ->column (cid));
470+ _olap_data_convertor->reserve (source_schema ->num_columns ());
471+ for (size_t cid = 0 ; cid < source_schema ->num_columns (); cid++) {
472+ _olap_data_convertor->add_column_data_convertor (source_schema ->column (cid));
444473 }
445474 return Status::OK ();
446475}
447476
448477Status RowBinlogSourceDataWriter::prepare_by_source_block (
449478 const Block* block, size_t row_pos, size_t num_rows,
450479 std::vector<uint32_t >& partial_source_cids, Block* full_block) {
451- _converted_columns.resize (_normal_column_ids.size ());
480+ TabletSchemaSPtr tablet_schema = _opt.source .tablet_schema ;
481+ // OlapBlockDataConvertor and historical lookup are indexed by source cid, so keep
482+ // converted columns sparse over the source schema instead of row-binlog normal order.
483+ _converted_columns.assign (tablet_schema->num_columns (), nullptr );
452484
453485 // LOG(INFO) << block->dump_data(0, num_rows);
454486
455487 // convert column data from engine format to storage layer format
456488 size_t col_pos_in_block = 0 ;
457- TabletSchemaSPtr tablet_schema = _opt.source .tablet_schema ;
458- const auto & including_cids =
459- partial_source_cids.empty () ? _normal_column_ids : partial_source_cids;
460- for (auto & cid : including_cids) {
461- const ColumnWithTypeAndName& col = block->get_by_position (col_pos_in_block++);
489+ const bool is_partial_update = !partial_source_cids.empty ();
490+ const size_t column_count =
491+ is_partial_update ? partial_source_cids.size () : _normal_column_ids.size ();
492+ for (size_t ordinal = 0 ; ordinal < column_count; ++ordinal) {
493+ const uint32_t source_cid =
494+ is_partial_update ? partial_source_cids[ordinal] : _normal_column_ids[ordinal];
495+ DCHECK_LT (source_cid, tablet_schema->num_columns ());
496+ const ColumnWithTypeAndName& col =
497+ block->get_by_position (is_partial_update ? col_pos_in_block++ : source_cid);
462498
463499 RETURN_IF_ERROR (_olap_data_convertor->set_source_content_with_specifid_column (
464- col, row_pos, num_rows, cid ));
500+ col, row_pos, num_rows, source_cid ));
465501 // olap data convertor alway start from id = 0
466- auto converted_result = _olap_data_convertor->convert_column_data (cid );
502+ auto converted_result = _olap_data_convertor->convert_column_data (source_cid );
467503 if (!converted_result.first .ok ()) {
468504 return converted_result.first ;
469505 }
470- _converted_columns[cid ] = converted_result.second ;
506+ _converted_columns[source_cid ] = converted_result.second ;
471507
472- if (cid < tablet_schema->num_key_columns ()) {
473- _key_columns.push_back (converted_result.second );
508+ full_block->replace_by_position (source_cid, col.column );
509+ }
510+ for (uint32_t cid : _normal_column_ids) {
511+ if (!tablet_schema->column (cid).is_key ()) {
512+ continue ;
474513 }
475- full_block->replace_by_position (cid, col.column );
514+ DCHECK_LT (cid, _converted_columns.size ());
515+ DCHECK (_converted_columns[cid] != nullptr );
516+ _key_columns.push_back (_converted_columns[cid]);
476517 }
477518 _num_rows = num_rows;
478519
@@ -497,15 +538,20 @@ Status RowBinlogSourceDataWriter::fill_normal_columns(
497538 std::vector<uint32_t >& partial_source_cids) {
498539 DCHECK_EQ (end - start, _normal_column_ids.size ());
499540
500- const auto & including_cids =
501- partial_source_cids.empty () ? _normal_column_ids : partial_source_cids;
502- for (size_t cid : including_cids) {
503- DCHECK (column_writers[start + cid]->get_column ()->type () ==
541+ const bool is_partial_update = !partial_source_cids.empty ();
542+ const size_t column_count =
543+ is_partial_update ? partial_source_cids.size () : _normal_column_ids.size ();
544+ for (size_t ordinal = 0 ; ordinal < column_count; ++ordinal) {
545+ uint32_t cid =
546+ is_partial_update ? partial_source_cids[ordinal] : _normal_column_ids[ordinal];
547+ size_t normal_ordinal = is_partial_update ? normal_column_ordinal (cid) : ordinal;
548+ DCHECK (column_writers[start + normal_ordinal]->get_column ()->type () ==
504549 _opt.source .tablet_schema ->columns ()[cid]->type ())
505550 << cid;
506- RETURN_IF_ERROR (column_writers[start + cid]->append (_converted_columns[cid]->get_nullmap (),
507- _converted_columns[cid]->get_data (),
508- _num_rows));
551+ DCHECK (_converted_columns[cid] != nullptr );
552+ RETURN_IF_ERROR (column_writers[start + normal_ordinal]->append (
553+ _converted_columns[cid]->get_nullmap (), _converted_columns[cid]->get_data (),
554+ _num_rows));
509555 }
510556
511557 return Status::OK ();
0 commit comments