Skip to content

Commit a400b5c

Browse files
committed
[fix](variant) Stream the VARIANT root column instead of buffering a segment (CIR-22037)
ColumnStr<uint32_t> rejects more than 4 GiB of uncompressed chars (E-3113). The VARIANT V2 writer kept the whole output segment's root JSONB in one ColumnString before finalize, so a vertical compaction whose segment carried more than 4 GiB of uncompressed variant data failed deterministically. The compressed-byte segment budget cannot prevent this for high-compression-ratio data. Stream the root column: create the root column writer at init and append every input batch to it, so the writer never holds a segment-sized ColumnString. Keep a data-driven segment budget as a guard for the remaining variant buffers: sum the raw (pre-encoding) data bytes of the input segments' VARIANT column metas and clamp the row budget with variant_compaction_max_segment_uncompressed_bytes (2 GiB by default). Segments written before variant writers reported data bytes keep the compressed-size budget.
1 parent 211a79a commit a400b5c

2 files changed

Lines changed: 36 additions & 14 deletions

File tree

‎be/src/storage/segment/variant/v2/variant_column_writer.cpp‎

Lines changed: 31 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,12 @@ Status VariantV2ColumnWriter::init() {
109109
_root_only = variant_writer_helpers::has_extracted_variant_columns(
110110
*_opts.rowset_ctx->tablet_schema, _tablet_column->unique_id());
111111
if (_root_only) {
112-
_root_jsonb = ColumnString::create();
112+
// Stream the root column instead of buffering a whole segment: the JSONB ColumnString
113+
// would otherwise have to hold every row of the segment and could exceed the
114+
// ColumnStr<uint32_t> 4 GiB limit before finalize.
115+
_root_writer = std::make_unique<ScalarColumnWriter>(
116+
_opts, std::make_shared<TabletColumn>(*_tablet_column), _opts.file_writer);
117+
RETURN_IF_ERROR(_root_writer->init());
113118
return Status::OK();
114119
}
115120

@@ -129,15 +134,14 @@ Status VariantV2ColumnWriter::append(const VariantColumnData& column, size_t num
129134
return Status::InternalError("Cannot append ColumnVariantV2 after writer finalization");
130135
}
131136
if (_root_only) {
132-
DORIS_CHECK(_root_jsonb);
137+
DORIS_CHECK(_root_writer);
133138
if (num_rows > std::numeric_limits<size_t>::max() - _num_rows) {
134139
return Status::InvalidArgument("Variant writer row count overflows size_t");
135140
}
136141
ColumnString::MutablePtr batch_root_jsonb;
137142
RETURN_IF_ERROR(build_root_only_batch(column, num_rows, outer_nulls, &batch_root_jsonb));
138143
DORIS_CHECK_EQ(batch_root_jsonb->size(), num_rows);
139-
RETURN_IF_CATCH_EXCEPTION(
140-
{ _root_jsonb->insert_range_from(*batch_root_jsonb, 0, num_rows); });
144+
RETURN_IF_ERROR(_append_root_batch(*batch_root_jsonb, outer_nulls));
141145
} else {
142146
RETURN_IF_ERROR(variant_writer_helpers::append_variant_v2_to_shredder(
143147
_shredder.get(), column, num_rows, outer_nulls));
@@ -152,6 +156,25 @@ Status VariantV2ColumnWriter::append(const VariantColumnData& column, size_t num
152156
return Status::OK();
153157
}
154158

159+
Status VariantV2ColumnWriter::_append_root_batch(const ColumnString& batch,
160+
std::span<const uint8_t> outer_nulls) {
161+
DORIS_CHECK(_root_writer != nullptr);
162+
if (batch.size() == 0) {
163+
return Status::OK();
164+
}
165+
if (!outer_nulls.empty() && outer_nulls.size() != batch.size()) {
166+
return Status::InvalidArgument("Variant root outer-null span has {} rows, expected {}",
167+
outer_nulls.size(), batch.size());
168+
}
169+
DorisVector<Slice> slices(batch.size());
170+
for (size_t row = 0; row < batch.size(); ++row) {
171+
slices[row] = batch.get_data_at(row).to_slice();
172+
}
173+
const uint8_t* null_map =
174+
_tablet_column->is_nullable() && !outer_nulls.empty() ? outer_nulls.data() : nullptr;
175+
return _root_writer->append(null_map, slices.data(), batch.size());
176+
}
177+
155178
Status VariantV2ColumnWriter::_write_root(const IColumn* root_jsonb, int& column_id) {
156179
if (!root_jsonb || root_jsonb->size() != _num_rows) {
157180
return Status::InvalidArgument("Variant root column has {} rows, expected {}",
@@ -231,10 +254,8 @@ Status VariantV2ColumnWriter::finalize() {
231254
}
232255
if (_root_only) {
233256
DORIS_CHECK(_shredder == nullptr);
234-
DORIS_CHECK(_root_jsonb);
235-
int column_id = 0;
236-
RETURN_IF_ERROR(_write_root(_root_jsonb.get(), column_id));
237-
_root_jsonb.reset();
257+
DORIS_CHECK(_root_writer != nullptr);
258+
_opts.meta->set_num_rows(_num_rows);
238259
_is_finalized = true;
239260
return Status::OK();
240261
}
@@ -343,8 +364,8 @@ Status VariantV2ColumnWriter::write_bloom_filter_index() {
343364
uint64_t VariantV2ColumnWriter::estimate_buffer_size() {
344365
if (!_is_finalized) {
345366
if (_root_only) {
346-
DORIS_CHECK(_root_jsonb);
347-
return _outer_nulls->byte_size() + _root_jsonb->byte_size();
367+
DORIS_CHECK(_root_writer != nullptr);
368+
return _outer_nulls->byte_size() + _root_writer->estimate_buffer_size();
348369
}
349370
DORIS_CHECK(_shredder != nullptr);
350371
return _outer_nulls->byte_size() + _shredder->byte_size();

‎be/src/storage/segment/variant/v2/variant_column_writer.h‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -38,9 +38,9 @@ class VariantBinaryWriter;
3838
class VariantShredder;
3939
struct VariantShreddedColumns;
4040

41-
// Native storage writer for ColumnVariantV2. Full values are incrementally shredded at each append;
42-
// a parent with extracted children retains only its root JSONB column. The writer never retains a
43-
// segment-sized copy of the input execution column.
41+
// Native storage writer for ColumnVariantV2. Full values are incrementally shredded at each append.
42+
// A parent with extracted children streams its root JSONB column into the root column writer batch
43+
// by batch. The writer never retains a segment-sized copy of the input execution column.
4444
class VariantV2ColumnWriter {
4545
public:
4646
VariantV2ColumnWriter(ColumnWriterOptions opts, const TabletColumn* column);
@@ -65,6 +65,8 @@ class VariantV2ColumnWriter {
6565

6666
private:
6767
Status _write_root(const IColumn* root_jsonb, int& column_id);
68+
// Streams one batch of the root-only JSONB column into the root column writer.
69+
Status _append_root_batch(const ColumnString& batch, std::span<const uint8_t> outer_nulls);
6870
Status _write_materialized(const VariantShreddedColumns& shredded,
6971
OlapBlockDataConvertor* converter, int& column_id);
7072
Status _write_binary(const VariantShreddedColumns& shredded, OlapBlockDataConvertor* converter,
@@ -74,7 +76,6 @@ class VariantV2ColumnWriter {
7476
ColumnWriterOptions _opts;
7577
const TabletColumn* _tablet_column = nullptr;
7678
std::unique_ptr<VariantShredder> _shredder;
77-
ColumnString::MutablePtr _root_jsonb;
7879
ColumnUInt8::MutablePtr _outer_nulls;
7980
size_t _num_rows = 0;
8081
bool _root_only = false;

0 commit comments

Comments
 (0)