Skip to content

Commit 92db91d

Browse files
committed
[fix](variant) Chunk VariantSubcolumnWriter value buffering (CIR-22037)
VariantSubcolumnWriter accumulated every value of an output segment in one VariantPathBuilder column, and materialized a second segment-sized dense column in finalize(). A VARIANT-typed extracted subcolumn whose non-null payload exceeds 4 GiB therefore always failed compaction with E-3113. Accumulate values into bounded chunks (256 MiB / 4M values) instead. Each chunk keeps the inferred type; finalize() merges the chunk types with variant_path_least_common_type(), converts every chunk to the final storage type and flushes it to the underlying writer independently, so neither the builder nor the dense materialization ever holds a segment-sized ColumnString.
1 parent 7b0958d commit 92db91d

4 files changed

Lines changed: 116 additions & 39 deletions

File tree

‎be/src/storage/segment/column_writer.h‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -651,12 +651,25 @@ class VariantSubcolumnWriter : public ColumnWriter {
651651
std::span<const uint8_t> outer_nulls);
652652
Status _ensure_input_format(const VariantColumnData& column);
653653
Status _initialize_v2_builder();
654+
Status _seal_chunk();
655+
// Bounded value chunks; each chunk is finalized and flushed independently so the writer never
656+
// holds a segment-sized ColumnString.
657+
struct ValueChunk {
658+
std::unique_ptr<VariantPathBuilder> builder;
659+
size_t row_begin = 0;
660+
size_t rows = 0;
661+
};
662+
static constexpr size_t kChunkBytes = 256ULL << 20;
663+
static constexpr size_t kChunkValues = 4ULL << 20;
654664
bool is_finalized() const;
655665
bool _is_finalized = false;
656666
ordinal_t _next_rowid = 0;
657667
size_t none_null_size = 0;
658668
VariantWriterInputFormat _input_format = VariantWriterInputFormat::UNSET;
659669
std::unique_ptr<VariantPathBuilder> _v2_builder;
670+
std::vector<ValueChunk> _chunks;
671+
size_t _chunk_row_begin = 0;
672+
size_t _chunk_rows = 0;
660673
size_t _num_rows = 0;
661674
ColumnWriterOptions _opts;
662675
std::unique_ptr<ColumnWriter> _writer;

‎be/src/storage/segment/variant/v2/variant_path_builder.cpp‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1038,6 +1038,10 @@ bool cast_introduced_null(const IColumn& source, const IColumn& target) {
10381038

10391039
} // namespace
10401040

1041+
DataTypePtr variant_path_least_common_type(const DataTypePtr& left, const DataTypePtr& right) {
1042+
return path_least_common_type(left, right);
1043+
}
1044+
10411045
DataTypePtr normalize_variant_path_integer_widths(const DataTypePtr& type) {
10421046
const DataTypePtr base = remove_nullable(type);
10431047
if (const auto* array = typeid_cast<const DataTypeArray*>(base.get())) {

‎be/src/storage/segment/variant/v2/variant_path_builder.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,4 +107,7 @@ VariantPathSelection select_variant_paths(std::span<const VariantPathSelectionCa
107107
size_t max_dynamic_materialized_paths,
108108
bool typed_paths_to_sparse);
109109

110+
// Least common type for a variant path. Used to merge types inferred by independent value chunks.
111+
DataTypePtr variant_path_least_common_type(const DataTypePtr& left, const DataTypePtr& right);
112+
110113
} // namespace doris::segment_v2

‎be/src/storage/segment/variant/variant_column_writer_impl.cpp‎

Lines changed: 96 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -587,7 +587,6 @@ Status VariantSubcolumnWriter::_ensure_input_format(const VariantColumnData& col
587587

588588
Status VariantSubcolumnWriter::_append_v2(const VariantColumnData& column, size_t num_rows,
589589
std::span<const uint8_t> outer_nulls) {
590-
DORIS_CHECK(_v2_builder != nullptr);
591590
DORIS_CHECK(column.column_data != nullptr);
592591
const auto* source = check_and_get_column<ColumnVariantV2>(*column.column_data);
593592
DORIS_CHECK(source != nullptr);
@@ -596,6 +595,11 @@ Status VariantSubcolumnWriter::_append_v2(const VariantColumnData& column, size_
596595
column.row_pos, column.row_pos + num_rows, source->size());
597596
}
598597
DORIS_CHECK(outer_nulls.empty() || outer_nulls.size() == num_rows);
598+
if (_v2_builder == nullptr) {
599+
RETURN_IF_ERROR(_initialize_v2_builder());
600+
_chunk_row_begin = _num_rows;
601+
_chunk_rows = 0;
602+
}
599603

600604
const auto append_encoded = [&](const ColumnVariantV2::ReadView& view, size_t begin) -> Status {
601605
DORIS_CHECK(!view.is_typed());
@@ -606,23 +610,40 @@ Status VariantSubcolumnWriter::_append_v2(const VariantColumnData& column, size_
606610
const size_t input_row = begin + offset;
607611
const VariantRef value = view.value_at(input_row);
608612
if (!value.is_null()) {
609-
RETURN_IF_ERROR(_v2_builder->append(value, _num_rows + offset));
613+
RETURN_IF_ERROR(_v2_builder->append(value, _num_rows + offset - _chunk_row_begin));
610614
}
611615
}
612616
return Status::OK();
613617
};
614618

615619
const auto view = source->read_view();
616620
if (!view.is_typed()) {
617-
return append_encoded(view, column.row_pos);
621+
RETURN_IF_ERROR(append_encoded(view, column.row_pos));
622+
} else {
623+
auto encoded_batch = ColumnVariantV2::create();
624+
RETURN_IF_CATCH_EXCEPTION({
625+
encoded_batch->insert_range_from(*source, column.row_pos, num_rows);
626+
encoded_batch->ensure_encoded();
627+
});
628+
RETURN_IF_ERROR(append_encoded(encoded_batch->read_view(), 0));
629+
}
630+
_chunk_rows = _num_rows + num_rows - _chunk_row_begin;
631+
if (_v2_builder->byte_size() >= kChunkBytes || _v2_builder->non_null_rows() >= kChunkValues) {
632+
RETURN_IF_ERROR(_seal_chunk());
618633
}
634+
return Status::OK();
635+
}
619636

620-
auto encoded_batch = ColumnVariantV2::create();
621-
RETURN_IF_CATCH_EXCEPTION({
622-
encoded_batch->insert_range_from(*source, column.row_pos, num_rows);
623-
encoded_batch->ensure_encoded();
624-
});
625-
return append_encoded(encoded_batch->read_view(), 0);
637+
Status VariantSubcolumnWriter::_seal_chunk() {
638+
if (_v2_builder == nullptr) {
639+
return Status::OK();
640+
}
641+
RETURN_IF_ERROR(_v2_builder->complete_rows(_chunk_rows));
642+
_chunks.push_back(ValueChunk {
643+
.builder = std::move(_v2_builder), .row_begin = _chunk_row_begin, .rows = _chunk_rows});
644+
_v2_builder.reset();
645+
_chunk_rows = 0;
646+
return Status::OK();
626647
}
627648

628649
Status VariantSubcolumnWriter::_append(const uint8_t* null_map, const uint8_t** ptr,
@@ -652,7 +673,11 @@ uint64_t VariantSubcolumnWriter::estimate_buffer_size() {
652673
if (is_finalized()) {
653674
return _writer ? _writer->estimate_buffer_size() : 0;
654675
}
655-
return _v2_builder ? _v2_builder->byte_size() : 0;
676+
size_t bytes = _v2_builder ? _v2_builder->byte_size() : 0;
677+
for (const auto& chunk : _chunks) {
678+
bytes += chunk.builder->byte_size();
679+
}
680+
return bytes;
656681
}
657682

658683
bool VariantSubcolumnWriter::is_finalized() const {
@@ -665,51 +690,63 @@ Status VariantSubcolumnWriter::finalize() {
665690
}
666691
if (_input_format == VariantWriterInputFormat::UNSET) {
667692
RETURN_IF_ERROR(_initialize_v2_builder());
693+
_chunk_row_begin = 0;
694+
_chunk_rows = 0;
668695
_input_format = VariantWriterInputFormat::V2;
669696
}
697+
RETURN_IF_ERROR(_seal_chunk());
698+
670699
const auto& parent_column =
671700
_opts.rowset_ctx->tablet_schema->column_by_uid(get_column()->parent_unique_id());
672701
DORIS_CHECK(!parent_column.variant_enable_nested_group());
673702

674-
DataTypePtr flush_type;
675-
ColumnPtr flush_values;
676-
DorisVector<uint32_t> flush_rowids;
677-
int64_t non_null_value_size = 0;
678-
DORIS_CHECK(_v2_builder != nullptr);
679-
RETURN_IF_ERROR(_v2_builder->complete_rows(_num_rows));
680703
DataTypePtr storage_type;
681704
TabletSchema::SubColumnInfo subcolumn_info;
705+
const PathInData relative_path = get_column()->path_info_ptr()->copy_pop_front();
682706
if (variant_util::generate_sub_column_info(*_opts.rowset_ctx->tablet_schema,
683-
parent_column.unique_id(),
684-
_v2_builder->path().get_path(), &subcolumn_info)) {
707+
parent_column.unique_id(), relative_path.get_path(),
708+
&subcolumn_info)) {
685709
storage_type = DataTypeFactory::instance().create_data_type(subcolumn_info.column);
686710
}
687711

688-
bool publish_builder = _v2_builder->non_null_rows() != 0;
712+
// Merge the types inferred by independent chunks; the result is what a single builder would
713+
// have inferred for the whole segment.
714+
DataTypePtr merged_type;
715+
bool has_values = false;
716+
for (auto& chunk : _chunks) {
717+
if (chunk.builder->non_null_rows() == 0) {
718+
continue;
719+
}
720+
has_values = true;
721+
RETURN_IF_ERROR(chunk.builder->convert_to(
722+
normalize_variant_path_integer_widths(chunk.builder->type())));
723+
merged_type = merged_type == nullptr
724+
? chunk.builder->type()
725+
: variant_path_least_common_type(merged_type, chunk.builder->type());
726+
}
727+
728+
bool publish_builder = has_values;
689729
if (publish_builder) {
690-
RETURN_IF_ERROR(_v2_builder->convert_to(
691-
normalize_variant_path_integer_widths(_v2_builder->type())));
692730
publish_builder =
693-
!variant_path_type_contains_nothing(_v2_builder->type()) ||
731+
!variant_path_type_contains_nothing(merged_type) ||
694732
(storage_type != nullptr && !variant_path_type_contains_nothing(storage_type));
695733
}
696-
if (publish_builder && storage_type != nullptr) {
697-
RETURN_IF_ERROR(_v2_builder->convert_to(storage_type));
698-
}
699734

735+
DataTypePtr flush_type;
736+
size_t non_null_value_size = 0;
700737
if (!publish_builder) {
701738
flush_type = DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_TINYINT,
702739
true /* is_nullable */);
703-
MutableColumnPtr defaults = flush_type->create_column();
704-
flush_values = std::move(defaults);
705740
} else {
706-
flush_type = _v2_builder->type();
707-
flush_values = _v2_builder->column();
708-
const std::span<const uint32_t> rowids = _v2_builder->rowids();
709-
flush_rowids.assign(rowids.begin(), rowids.end());
710-
non_null_value_size = cast_set<int64_t>(flush_rowids.size());
741+
flush_type = storage_type != nullptr ? storage_type : merged_type;
742+
for (auto& chunk : _chunks) {
743+
if (chunk.builder->non_null_rows() == 0) {
744+
continue;
745+
}
746+
RETURN_IF_ERROR(chunk.builder->convert_to(flush_type));
747+
non_null_value_size += chunk.builder->rowids().size();
748+
}
711749
}
712-
_v2_builder.reset();
713750

714751
TabletColumn flush_column = variant_util::get_column_by_type(
715752
flush_type, get_column()->name(),
@@ -727,13 +764,33 @@ Status VariantSubcolumnWriter::finalize() {
727764
_indexes, &opts, non_null_value_size, need_record_none_null_value_size));
728765

729766
_opts = opts;
730-
OlapBlockDataConvertor converter;
731-
RETURN_IF_ERROR(variant_writer_helpers::append_sparse_converted_column(
732-
flush_column, _writer.get(), &converter, 0, flush_type, flush_values, flush_rowids,
733-
_num_rows));
767+
if (!publish_builder) {
768+
OlapBlockDataConvertor converter;
769+
MutableColumnPtr defaults = flush_type->create_column();
770+
RETURN_IF_ERROR(variant_writer_helpers::append_sparse_converted_column(
771+
flush_column, _writer.get(), &converter, 0, flush_type, std::move(defaults), {},
772+
_num_rows));
773+
} else {
774+
for (auto& chunk : _chunks) {
775+
// Each chunk owns one contiguous row range, so flushing it needs its own converter
776+
// slot. Reuse is not possible: append_sparse_converted_column registers a convertor
777+
// per call.
778+
OlapBlockDataConvertor converter;
779+
if (chunk.builder->non_null_rows() == 0) {
780+
MutableColumnPtr empty = flush_type->create_column();
781+
RETURN_IF_ERROR(variant_writer_helpers::append_sparse_converted_column(
782+
flush_column, _writer.get(), &converter, 0, flush_type, std::move(empty),
783+
{}, chunk.rows));
784+
continue;
785+
}
786+
RETURN_IF_ERROR(variant_writer_helpers::append_sparse_converted_column(
787+
flush_column, _writer.get(), &converter, 0, flush_type, chunk.builder->column(),
788+
chunk.builder->rowids(), chunk.rows));
789+
}
790+
}
734791
_opts.meta->set_num_rows(_num_rows);
735-
none_null_size = cast_set<size_t>(non_null_value_size);
736-
792+
none_null_size = non_null_value_size;
793+
_chunks.clear();
737794
_is_finalized = true;
738795
return Status::OK();
739796
}

0 commit comments

Comments
 (0)