Skip to content

Commit 9512c3c

Browse files
authored
[fix](binlog) wrap Row Binlog AFTER columns as nullable before conversion (#66553)
Issue Number: close #65951 Related PR: #65810 Problem Summary: Row Binlog makes normal non-key AFTER columns nullable even when their source columns are NOT NULL. `OlapBlockDataConvertor` derives the null map from the runtime source column, so a non-nullable complex source provides no top-level null map. Nullable ARRAY, MAP, and STRUCT writers can then leave their physical null child empty and abort during segment finalization. This PR keeps the fix local to Row Binlog: - Wrap normal non-key AFTER source columns as `ColumnNullable<original>` with an all-zero null map before conversion. - Keep key columns unchanged, preserve original source columns in `full_block`, and do not mutate the shared input Block. - For partial-update missing AFTER columns, wrap each column filled from the historical Rowset before conversion and append. - Restore the original generic `ColumnWriter::append()` behavior and remove the implicit null-map state, avoiding behavior changes in Scalar, VARIANT, and other common write paths. The test covers full updates, explicit partial updates, and partial updates whose NOT NULL MAP/STRUCT values are filled from a historical Rowset. It verifies logical readback and the physical top-level null-child row count. ### Release note Fix a BE crash when Row Binlog writes NOT NULL complex columns into nullable AFTER columns.
1 parent 634cbaa commit 9512c3c

3 files changed

Lines changed: 349 additions & 13 deletions

File tree

‎be/src/storage/segment/row_binlog_segment_writer.cpp‎

Lines changed: 29 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
#include "core/block/column_with_type_and_name.h"
2525
#include "core/column/column_nullable.h"
2626
#include "core/column/column_vector.h"
27+
#include "core/data_type/data_type_nullable.h"
2728
#include "storage/binlog.h"
2829
#include "storage/iterator/olap_data_convertor.h"
2930
#include "storage/olap_utils.h"
@@ -32,6 +33,17 @@
3233
namespace doris {
3334
namespace segment_v2 {
3435

36+
namespace {
37+
38+
ColumnWithTypeAndName make_nullable_row_binlog_value_column(const ColumnWithTypeAndName& source) {
39+
auto result = source;
40+
result.column = make_nullable(source.column);
41+
result.type = make_nullable(source.type);
42+
return result;
43+
}
44+
45+
} // namespace
46+
3547
RowBinlogSourceDataWriter::RowBinlogSourceDataWriter(const SegmentWriteBinlogOptions& opt)
3648
: _opt(opt) {}
3749

@@ -208,9 +220,9 @@ Status RowBinlogSegmentWriter::append_block(const Block* block, size_t row_pos,
208220
std::vector<uint32_t> row_binlog_missing_column_ids;
209221
row_binlog_missing_column_ids.reserve(
210222
_binlog_opts.source.partial_update_info->missing_cids.size());
211-
// Missing cids are source cids too. Only normal columns have AFTER writers.
223+
// Missing cids are source cids. Only Row Binlog value columns have AFTER writers.
212224
for (uint32_t cid : _binlog_opts.source.partial_update_info->missing_cids) {
213-
if (_source_data_writer->is_normal_column(cid)) {
225+
if (_source_data_writer->is_row_binlog_value_column(cid)) {
214226
row_binlog_missing_column_ids.emplace_back(cid);
215227
}
216228
}
@@ -220,9 +232,11 @@ Status RowBinlogSegmentWriter::append_block(const Block* block, size_t row_pos,
220232

221233
// write AFTER missing columns from full_block to segment
222234
auto& after_convertor = _source_data_writer->olap_data_convertor();
223-
RETURN_IF_ERROR(after_convertor->set_source_content_with_specifid_columns(
224-
&full_block, row_pos, num_rows, row_binlog_missing_column_ids));
225235
for (auto cid : row_binlog_missing_column_ids) {
236+
const auto nullable_source_column_for_after =
237+
make_nullable_row_binlog_value_column(full_block.get_by_position(cid));
238+
RETURN_IF_ERROR(after_convertor->set_source_content_with_specifid_column(
239+
nullable_source_column_for_after, row_pos, num_rows, cid));
226240
auto converted_cid =
227241
_normal_col_start_id + _source_data_writer->normal_column_ordinal(cid);
228242
auto converted_result = after_convertor->convert_column_data(cid);
@@ -449,6 +463,12 @@ bool RowBinlogSourceDataWriter::is_normal_column(uint32_t source_cid) const {
449463
_normal_column_ids.end();
450464
}
451465

466+
bool RowBinlogSourceDataWriter::is_row_binlog_value_column(uint32_t source_cid) const {
467+
DCHECK(_opt.source.tablet_schema != nullptr);
468+
DCHECK_LT(source_cid, _opt.source.tablet_schema->num_columns());
469+
return is_normal_column(source_cid) && !_opt.source.tablet_schema->column(source_cid).is_key();
470+
}
471+
452472
size_t RowBinlogSourceDataWriter::normal_column_ordinal(uint32_t source_cid) const {
453473
auto it = std::find(_normal_column_ids.begin(), _normal_column_ids.end(), source_cid);
454474
DCHECK(it != _normal_column_ids.end()) << source_cid;
@@ -499,16 +519,18 @@ Status RowBinlogSourceDataWriter::prepare_by_source_block(
499519
const ColumnWithTypeAndName& col =
500520
block->get_by_position(is_partial_update ? col_pos_in_block++ : source_cid);
501521

522+
full_block->replace_by_position(source_cid, col.column);
523+
const auto source_column_for_after = is_row_binlog_value_column(source_cid)
524+
? make_nullable_row_binlog_value_column(col)
525+
: col;
502526
RETURN_IF_ERROR(_olap_data_convertor->set_source_content_with_specifid_column(
503-
col, row_pos, num_rows, source_cid));
527+
source_column_for_after, row_pos, num_rows, source_cid));
504528
// olap data convertor alway start from id = 0
505529
auto converted_result = _olap_data_convertor->convert_column_data(source_cid);
506530
if (!converted_result.first.ok()) {
507531
return converted_result.first;
508532
}
509533
_converted_columns[source_cid] = converted_result.second;
510-
511-
full_block->replace_by_position(source_cid, col.column);
512534
}
513535
for (uint32_t cid : _normal_column_ids) {
514536
if (!tablet_schema->column(cid).is_key()) {

‎be/src/storage/segment/row_binlog_segment_writer.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,8 @@ class RowBinlogSourceDataWriter {
5555

5656
bool is_normal_column(uint32_t source_cid) const;
5757

58+
bool is_row_binlog_value_column(uint32_t source_cid) const;
59+
5860
size_t normal_column_ordinal(uint32_t source_cid) const;
5961

6062
private:

0 commit comments

Comments
 (0)