@@ -587,7 +587,6 @@ Status VariantSubcolumnWriter::_ensure_input_format(const VariantColumnData& col
587587
588588Status VariantSubcolumnWriter::_append_v2 (const VariantColumnData& column, size_t num_rows,
589589 std::span<const uint8_t > outer_nulls) {
590- DORIS_CHECK (_v2_builder != nullptr );
591590 DORIS_CHECK (column.column_data != nullptr );
592591 const auto * source = check_and_get_column<ColumnVariantV2>(*column.column_data );
593592 DORIS_CHECK (source != nullptr );
@@ -596,6 +595,11 @@ Status VariantSubcolumnWriter::_append_v2(const VariantColumnData& column, size_
596595 column.row_pos , column.row_pos + num_rows, source->size ());
597596 }
598597 DORIS_CHECK (outer_nulls.empty () || outer_nulls.size () == num_rows);
598+ if (_v2_builder == nullptr ) {
599+ RETURN_IF_ERROR (_initialize_v2_builder ());
600+ _chunk_row_begin = _num_rows;
601+ _chunk_rows = 0 ;
602+ }
599603
600604 const auto append_encoded = [&](const ColumnVariantV2::ReadView& view, size_t begin) -> Status {
601605 DORIS_CHECK (!view.is_typed ());
@@ -606,23 +610,40 @@ Status VariantSubcolumnWriter::_append_v2(const VariantColumnData& column, size_
606610 const size_t input_row = begin + offset;
607611 const VariantRef value = view.value_at (input_row);
608612 if (!value.is_null ()) {
609- RETURN_IF_ERROR (_v2_builder->append (value, _num_rows + offset));
613+ RETURN_IF_ERROR (_v2_builder->append (value, _num_rows + offset - _chunk_row_begin ));
610614 }
611615 }
612616 return Status::OK ();
613617 };
614618
615619 const auto view = source->read_view ();
616620 if (!view.is_typed ()) {
617- return append_encoded (view, column.row_pos );
621+ RETURN_IF_ERROR (append_encoded (view, column.row_pos ));
622+ } else {
623+ auto encoded_batch = ColumnVariantV2::create ();
624+ RETURN_IF_CATCH_EXCEPTION ({
625+ encoded_batch->insert_range_from (*source, column.row_pos , num_rows);
626+ encoded_batch->ensure_encoded ();
627+ });
628+ RETURN_IF_ERROR (append_encoded (encoded_batch->read_view (), 0 ));
629+ }
630+ _chunk_rows = _num_rows + num_rows - _chunk_row_begin;
631+ if (_v2_builder->byte_size () >= kChunkBytes || _v2_builder->non_null_rows () >= kChunkValues ) {
632+ RETURN_IF_ERROR (_seal_chunk ());
618633 }
634+ return Status::OK ();
635+ }
619636
620- auto encoded_batch = ColumnVariantV2::create ();
621- RETURN_IF_CATCH_EXCEPTION ({
622- encoded_batch->insert_range_from (*source, column.row_pos , num_rows);
623- encoded_batch->ensure_encoded ();
624- });
625- return append_encoded (encoded_batch->read_view (), 0 );
637+ Status VariantSubcolumnWriter::_seal_chunk () {
638+ if (_v2_builder == nullptr ) {
639+ return Status::OK ();
640+ }
641+ RETURN_IF_ERROR (_v2_builder->complete_rows (_chunk_rows));
642+ _chunks.push_back (ValueChunk {
643+ .builder = std::move (_v2_builder), .row_begin = _chunk_row_begin, .rows = _chunk_rows});
644+ _v2_builder.reset ();
645+ _chunk_rows = 0 ;
646+ return Status::OK ();
626647}
627648
628649Status VariantSubcolumnWriter::_append (const uint8_t * null_map, const uint8_t ** ptr,
@@ -652,7 +673,11 @@ uint64_t VariantSubcolumnWriter::estimate_buffer_size() {
652673 if (is_finalized ()) {
653674 return _writer ? _writer->estimate_buffer_size () : 0 ;
654675 }
655- return _v2_builder ? _v2_builder->byte_size () : 0 ;
676+ size_t bytes = _v2_builder ? _v2_builder->byte_size () : 0 ;
677+ for (const auto & chunk : _chunks) {
678+ bytes += chunk.builder ->byte_size ();
679+ }
680+ return bytes;
656681}
657682
658683bool VariantSubcolumnWriter::is_finalized () const {
@@ -665,51 +690,63 @@ Status VariantSubcolumnWriter::finalize() {
665690 }
666691 if (_input_format == VariantWriterInputFormat::UNSET ) {
667692 RETURN_IF_ERROR (_initialize_v2_builder ());
693+ _chunk_row_begin = 0 ;
694+ _chunk_rows = 0 ;
668695 _input_format = VariantWriterInputFormat::V2 ;
669696 }
697+ RETURN_IF_ERROR (_seal_chunk ());
698+
670699 const auto & parent_column =
671700 _opts.rowset_ctx ->tablet_schema ->column_by_uid (get_column ()->parent_unique_id ());
672701 DORIS_CHECK (!parent_column.variant_enable_nested_group ());
673702
674- DataTypePtr flush_type;
675- ColumnPtr flush_values;
676- DorisVector<uint32_t > flush_rowids;
677- int64_t non_null_value_size = 0 ;
678- DORIS_CHECK (_v2_builder != nullptr );
679- RETURN_IF_ERROR (_v2_builder->complete_rows (_num_rows));
680703 DataTypePtr storage_type;
681704 TabletSchema::SubColumnInfo subcolumn_info;
705+ const PathInData relative_path = get_column ()->path_info_ptr ()->copy_pop_front ();
682706 if (variant_util::generate_sub_column_info (*_opts.rowset_ctx ->tablet_schema ,
683- parent_column.unique_id (),
684- _v2_builder-> path (). get_path (), &subcolumn_info)) {
707+ parent_column.unique_id (), relative_path. get_path (),
708+ &subcolumn_info)) {
685709 storage_type = DataTypeFactory::instance ().create_data_type (subcolumn_info.column );
686710 }
687711
688- bool publish_builder = _v2_builder->non_null_rows () != 0 ;
712+ // Merge the types inferred by independent chunks; the result is what a single builder would
713+ // have inferred for the whole segment.
714+ DataTypePtr merged_type;
715+ bool has_values = false ;
716+ for (auto & chunk : _chunks) {
717+ if (chunk.builder ->non_null_rows () == 0 ) {
718+ continue ;
719+ }
720+ has_values = true ;
721+ RETURN_IF_ERROR (chunk.builder ->convert_to (
722+ normalize_variant_path_integer_widths (chunk.builder ->type ())));
723+ merged_type = merged_type == nullptr
724+ ? chunk.builder ->type ()
725+ : variant_path_least_common_type (merged_type, chunk.builder ->type ());
726+ }
727+
728+ bool publish_builder = has_values;
689729 if (publish_builder) {
690- RETURN_IF_ERROR (_v2_builder->convert_to (
691- normalize_variant_path_integer_widths (_v2_builder->type ())));
692730 publish_builder =
693- !variant_path_type_contains_nothing (_v2_builder-> type () ) ||
731+ !variant_path_type_contains_nothing (merged_type ) ||
694732 (storage_type != nullptr && !variant_path_type_contains_nothing (storage_type));
695733 }
696- if (publish_builder && storage_type != nullptr ) {
697- RETURN_IF_ERROR (_v2_builder->convert_to (storage_type));
698- }
699734
735+ DataTypePtr flush_type;
736+ size_t non_null_value_size = 0 ;
700737 if (!publish_builder) {
701738 flush_type = DataTypeFactory::instance ().create_data_type (PrimitiveType::TYPE_TINYINT ,
702739 true /* is_nullable */ );
703- MutableColumnPtr defaults = flush_type->create_column ();
704- flush_values = std::move (defaults);
705740 } else {
706- flush_type = _v2_builder->type ();
707- flush_values = _v2_builder->column ();
708- const std::span<const uint32_t > rowids = _v2_builder->rowids ();
709- flush_rowids.assign (rowids.begin (), rowids.end ());
710- non_null_value_size = cast_set<int64_t >(flush_rowids.size ());
741+ flush_type = storage_type != nullptr ? storage_type : merged_type;
742+ for (auto & chunk : _chunks) {
743+ if (chunk.builder ->non_null_rows () == 0 ) {
744+ continue ;
745+ }
746+ RETURN_IF_ERROR (chunk.builder ->convert_to (flush_type));
747+ non_null_value_size += chunk.builder ->rowids ().size ();
748+ }
711749 }
712- _v2_builder.reset ();
713750
714751 TabletColumn flush_column = variant_util::get_column_by_type (
715752 flush_type, get_column ()->name (),
@@ -727,13 +764,33 @@ Status VariantSubcolumnWriter::finalize() {
727764 _indexes, &opts, non_null_value_size, need_record_none_null_value_size));
728765
729766 _opts = opts;
730- OlapBlockDataConvertor converter;
731- RETURN_IF_ERROR (variant_writer_helpers::append_sparse_converted_column (
732- flush_column, _writer.get (), &converter, 0 , flush_type, flush_values, flush_rowids,
733- _num_rows));
767+ if (!publish_builder) {
768+ OlapBlockDataConvertor converter;
769+ MutableColumnPtr defaults = flush_type->create_column ();
770+ RETURN_IF_ERROR (variant_writer_helpers::append_sparse_converted_column (
771+ flush_column, _writer.get (), &converter, 0 , flush_type, std::move (defaults), {},
772+ _num_rows));
773+ } else {
774+ for (auto & chunk : _chunks) {
775+ // Each chunk owns one contiguous row range, so flushing it needs its own converter
776+ // slot. Reuse is not possible: append_sparse_converted_column registers a convertor
777+ // per call.
778+ OlapBlockDataConvertor converter;
779+ if (chunk.builder ->non_null_rows () == 0 ) {
780+ MutableColumnPtr empty = flush_type->create_column ();
781+ RETURN_IF_ERROR (variant_writer_helpers::append_sparse_converted_column (
782+ flush_column, _writer.get (), &converter, 0 , flush_type, std::move (empty),
783+ {}, chunk.rows ));
784+ continue ;
785+ }
786+ RETURN_IF_ERROR (variant_writer_helpers::append_sparse_converted_column (
787+ flush_column, _writer.get (), &converter, 0 , flush_type, chunk.builder ->column (),
788+ chunk.builder ->rowids (), chunk.rows ));
789+ }
790+ }
734791 _opts.meta ->set_num_rows (_num_rows);
735- none_null_size = cast_set< size_t >( non_null_value_size) ;
736-
792+ none_null_size = non_null_value_size;
793+ _chunks. clear ();
737794 _is_finalized = true ;
738795 return Status::OK ();
739796}
0 commit comments