Skip to content

Commit 720f7f7

Browse files
committed
[fix](variant) Keep chunk type inference and ARRAY chunk dispatch consistent (CIR-22037)
- A new chunk starts from the type inferred by the sealed chunks, so the single builder's JSONB fallback still applies across chunk boundaries (a bool after integers must not be coerced to 1/0 by the least-common-type merge). - All-NULL ARRAY chunks are flushed through the array writer, which updates the offsets, nulls and inverted index together; append_nulls only wrote the data column and left the index inconsistent. - One convertor per column in finalize: the ARRAY convertor carries the accumulated item offset, so chunked flushes keep absolute offsets. - Tests for int-then-bool across a boundary and an ARRAY all-NULL tail chunk.
1 parent 0bbaf06 commit 720f7f7

5 files changed

Lines changed: 156 additions & 15 deletions

File tree

‎be/src/storage/segment/column_writer.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -678,6 +678,9 @@ class VariantSubcolumnWriter : public ColumnWriter {
678678
std::vector<ValueChunk> _chunks;
679679
size_t _chunk_row_begin = 0;
680680
size_t _chunk_rows = 0;
681+
// Type accumulated by the sealed chunks. A new chunk starts from it so appends follow the
682+
// exact same inference and JSONB-fallback sequence as one uninterrupted builder.
683+
DataTypePtr _running_type;
681684
size_t _num_rows = 0;
682685
ColumnWriterOptions _opts;
683686
std::unique_ptr<ColumnWriter> _writer;

‎be/src/storage/segment/variant/variant_column_writer_impl.cpp‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -602,6 +602,12 @@ Status VariantSubcolumnWriter::_append_v2(const VariantColumnData& column, size_
602602
DORIS_CHECK(outer_nulls.empty() || outer_nulls.size() == num_rows);
603603
if (_v2_builder == nullptr) {
604604
RETURN_IF_ERROR(_initialize_v2_builder());
605+
if (_running_type != nullptr) {
606+
// Continue from the type inferred by the sealed chunks. Appending a value that the
607+
// previous chunks' type cannot hold (for example a bool after integers) then takes
608+
// the builder's own JSONB fallback inside this chunk instead of being merged away.
609+
RETURN_IF_ERROR(_v2_builder->convert_to(_running_type));
610+
}
605611
_chunk_row_begin = _num_rows;
606612
_chunk_rows = 0;
607613
}
@@ -656,6 +662,7 @@ Status VariantSubcolumnWriter::_seal_chunk() {
656662
return Status::OK();
657663
}
658664
RETURN_IF_ERROR(_v2_builder->complete_rows(_chunk_rows));
665+
_running_type = _v2_builder->type();
659666
_chunks.push_back(ValueChunk {
660667
.builder = std::move(_v2_builder), .row_begin = _chunk_row_begin, .rows = _chunk_rows});
661668
_v2_builder.reset();
@@ -803,18 +810,17 @@ Status VariantSubcolumnWriter::finalize() {
803810
_indexes, &opts, non_null_value_size, need_record_none_null_value_size));
804811

805812
_opts = opts;
813+
// One convertor for the whole column. The ARRAY convertor carries its accumulated item offset
814+
// across calls, so flushing chunk by chunk keeps the absolute array offsets correct; a fresh
815+
// convertor per chunk would restart them at zero and corrupt the column.
816+
OlapBlockDataConvertor converter;
806817
if (!publish_builder) {
807-
OlapBlockDataConvertor converter;
808818
MutableColumnPtr defaults = flush_type->create_column();
809819
RETURN_IF_ERROR(variant_writer_helpers::append_sparse_converted_column(
810820
flush_column, _writer.get(), &converter, 0, flush_type, std::move(defaults), {},
811821
_num_rows));
812822
} else {
813823
for (auto& chunk : _chunks) {
814-
// Each chunk owns one contiguous row range, so flushing it needs its own converter
815-
// slot. Reuse is not possible: append_sparse_converted_column registers a convertor
816-
// per call.
817-
OlapBlockDataConvertor converter;
818824
if (chunk.builder->non_null_rows() == 0) {
819825
MutableColumnPtr empty = flush_type->create_column();
820826
RETURN_IF_ERROR(variant_writer_helpers::append_sparse_converted_column(

‎be/src/storage/segment/variant/variant_writer_helpers.cpp‎

Lines changed: 27 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
#include "storage/segment/variant/variant_writer_helpers.h"
1919

2020
#include <algorithm>
21+
#include <atomic>
2122
#include <memory>
2223
#include <span>
2324
#include <string>
@@ -322,6 +323,20 @@ Status append_sparse_scalar_column(const TabletColumn& tablet_column, ColumnWrit
322323

323324
} // namespace
324325

326+
#ifdef BE_TEST
327+
namespace {
328+
std::atomic<uint64_t> g_array_all_null_chunks_for_test = 0;
329+
} // namespace
330+
331+
uint64_t array_all_null_chunks_for_test() {
332+
return g_array_all_null_chunks_for_test.load(std::memory_order_relaxed);
333+
}
334+
335+
void reset_array_all_null_chunks_for_test() {
336+
g_array_all_null_chunks_for_test.store(0, std::memory_order_relaxed);
337+
}
338+
#endif
339+
325340
Status append_sparse_converted_column(const TabletColumn& tablet_column, ColumnWriter* writer,
326341
OlapBlockDataConvertor* converter, int column_id,
327342
const DataTypePtr& type, const ColumnPtr& values_column,
@@ -338,16 +353,23 @@ Status append_sparse_converted_column(const TabletColumn& tablet_column, ColumnW
338353
if (total_rows == 0) {
339354
return Status::OK();
340355
}
341-
if (rowids.empty()) {
342-
DORIS_CHECK(tablet_column.is_nullable());
343-
return writer->append_nulls(total_rows);
344-
}
345-
346356
const DataTypePtr base_type = remove_nullable(type);
347357
if (base_type->get_primitive_type() == PrimitiveType::TYPE_ARRAY) {
358+
#ifdef BE_TEST
359+
if (rowids.empty()) {
360+
g_array_all_null_chunks_for_test.fetch_add(1, std::memory_order_relaxed);
361+
}
362+
#endif
363+
// ARRAY writers materialize a dense N-row column and update offsets, nulls and the
364+
// inverted index together. An all-NULL chunk (empty rowids) must take this path too:
365+
// append_nulls only writes column data and leaves the inverted index inconsistent.
348366
return append_sparse_array_column(tablet_column, writer, converter, column_id, type,
349367
values_column, rowids, total_rows);
350368
}
369+
if (rowids.empty()) {
370+
DORIS_CHECK(tablet_column.is_nullable());
371+
return writer->append_nulls(total_rows);
372+
}
351373
return append_sparse_scalar_column(tablet_column, writer, converter, column_id, type,
352374
values_column, rowids, total_rows);
353375
}

‎be/src/storage/segment/variant/variant_writer_helpers.h‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,13 @@ Status convert_and_write_column(OlapBlockDataConvertor* converter, const TabletC
7676

7777
// Converts only present values. Missing logical rows are appended as NULL; ARRAY uses a one-path
7878
// full-column fallback because its converted representation is not fixed-width/strided.
79+
#ifdef BE_TEST
80+
// Number of all-NULL ARRAY chunks written through the array path. The dispatch must never send
81+
// them to the scalar append_nulls fast path, which does not maintain the inverted index.
82+
uint64_t array_all_null_chunks_for_test();
83+
void reset_array_all_null_chunks_for_test();
84+
#endif
85+
7986
Status append_sparse_converted_column(const TabletColumn& tablet_column, ColumnWriter* writer,
8087
OlapBlockDataConvertor* converter, int column_id,
8188
const DataTypePtr& type, const ColumnPtr& values_column,

‎be/test/storage/variant/variant_column_writer_reader_test.cpp‎

Lines changed: 108 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@
4747
#include "core/value/uuid_value.h"
4848
#include "core/value/variant/variant_batch_builder.h"
4949
#include "core/value/variant/variant_parquet_encoding.h"
50+
#include "exec/common/variant_util.h"
5051
#include "exprs/function/search/function_search.h"
5152
#include "exprs/function/search/variant_inverted_index_search.h"
5253
#include "exprs/vectorized_fn_call.h"
@@ -79,6 +80,7 @@
7980
#include "storage/segment/variant/variant_column_reader.h"
8081
#include "storage/segment/variant/variant_column_writer_impl.h"
8182
#include "storage/segment/variant/variant_doc_snpashot_compact_iterator.h"
83+
#include "storage/segment/variant/variant_writer_helpers.h"
8284
#include "storage/storage_engine.h"
8385
#include "storage/tablet/tablet_manager.h"
8486
#include "testutil/variant_util.h"
@@ -1872,9 +1874,13 @@ class VariantColumnWriterReaderTest : public testing::Test {
18721874
RETURN_IF_ERROR(ColumnReader::create(reader_options, footer.columns(0), footer.num_rows(),
18731875
file_reader, &reader));
18741876

1875-
TabletColumn read_column = _tablet_schema->column(1);
1876-
read_column.set_type(static_cast<FieldType>(footer.columns(0).type()));
1877-
auto read_type = DataTypeFactory::instance().create_data_type(read_column, false);
1877+
// Build the read type and the matching tablet column from the footer meta so nested types
1878+
// (ARRAY/MAP/STRUCT) keep their child type information; the schema column only carries the
1879+
// top-level type and the array iterator needs the element sub-column.
1880+
auto read_type = DataTypeFactory::instance().create_data_type(footer.columns(0));
1881+
TabletColumn read_column = variant_util::get_column_by_type(
1882+
read_type, _tablet_schema->column(1).name(),
1883+
variant_util::ExtraInfo {.unique_id = -1, .parent_unique_id = 1});
18781884
ColumnIteratorUPtr iterator;
18791885
RETURN_IF_ERROR(reader->new_iterator(&iterator, &read_column));
18801886
OlapReaderStatistics stats;
@@ -2489,8 +2495,9 @@ TEST_F(VariantColumnWriterReaderTest, test_variant_root_writer_streams_per_batch
24892495
converter.clear_source_content(0);
24902496
max_buffered_bytes = std::max<uint64_t>(max_buffered_bytes, writer->estimate_buffer_size());
24912497
}
2492-
// The buffering writer retained the whole segment (kRows * kValueBytes ~ 16 MiB); the
2493-
// streaming writer holds at most the current page plus the outer-null map.
2498+
// The buffering writer retained the whole segment (kRows * kValueBytes ~ 16 MiB). The
2499+
// streaming writer flushes the root column batch by batch, so its own retained raw buffer is
2500+
// about one batch; encoded pages and the other variant column writers are separate.
24942501
EXPECT_LT(max_buffered_bytes, 2 * kBatchRows * kValueBytes);
24952502

24962503
ASSERT_TRUE(writer->finish().ok());
@@ -9252,4 +9259,100 @@ TEST_F(VariantColumnWriterReaderTest, test_variant_subcolumn_writer_chunks_by_by
92529259
EXPECT_EQ(chunked_rows, ref_rows);
92539260
}
92549261

9262+
TEST_F(VariantColumnWriterReaderTest, test_variant_subcolumn_writer_chunks_int_then_bool) {
9263+
// The single builder falls back to JSONB when a bool follows integers: appending `true` into
9264+
// an Int64 column is not a representable promotion. Chunks must carry the inferred type across
9265+
// their boundary so the same value sequence keeps the JSONB fallback instead of coercing
9266+
// true/false to 1/0 through the least-common-type merge.
9267+
init_extracted_variant_schema(11406);
9268+
const std::vector<std::string> jsons {"7", "true", "false", "8"};
9269+
ColumnPtr source;
9270+
DataTypePtr source_type;
9271+
ASSERT_TRUE(create_variant_writer_source(VariantWriterInput::V2, jsons, 0, false, {}, &source,
9272+
&source_type)
9273+
.ok());
9274+
9275+
SegmentFooterPB ref_footer;
9276+
std::string ref_path;
9277+
ASSERT_TRUE(write_extracted_variant_segment(source, source_type, "subcolumn_bool_ref",
9278+
&ref_footer, &ref_path)
9279+
.ok());
9280+
std::vector<std::string> ref_rows;
9281+
ASSERT_TRUE(read_extracted_variant_rows(ref_footer, ref_path, &ref_rows).ok());
9282+
9283+
struct ChunkGuard {
9284+
ChunkGuard() { VariantSubcolumnWriter::set_chunk_values_for_test(1); }
9285+
~ChunkGuard() { VariantSubcolumnWriter::set_chunk_values_for_test(0); }
9286+
} guard;
9287+
size_t max_chunks = 0;
9288+
SegmentFooterPB chunked_footer;
9289+
std::string chunked_path;
9290+
ASSERT_TRUE(write_extracted_variant_segment(
9291+
source, source_type, "subcolumn_bool_chunked", &chunked_footer,
9292+
&chunked_path, 0, 1,
9293+
[&](ColumnWriter* writer) {
9294+
auto* subcolumn = dynamic_cast<VariantSubcolumnWriter*>(writer);
9295+
ASSERT_NE(subcolumn, nullptr);
9296+
max_chunks = std::max(max_chunks, subcolumn->num_chunks_for_test());
9297+
})
9298+
.ok());
9299+
EXPECT_GE(max_chunks, 2);
9300+
9301+
std::vector<std::string> chunked_rows;
9302+
ASSERT_TRUE(read_extracted_variant_rows(chunked_footer, chunked_path, &chunked_rows).ok());
9303+
EXPECT_EQ(chunked_rows, ref_rows);
9304+
EXPECT_EQ(chunked_rows, (std::vector<std::string> {"7", "true", "false", "8"}));
9305+
EXPECT_EQ(static_cast<FieldType>(ref_footer.columns(0).type()),
9306+
FieldType::OLAP_FIELD_TYPE_JSONB);
9307+
EXPECT_EQ(static_cast<FieldType>(chunked_footer.columns(0).type()),
9308+
FieldType::OLAP_FIELD_TYPE_JSONB);
9309+
}
9310+
9311+
TEST_F(VariantColumnWriterReaderTest, test_variant_subcolumn_writer_chunks_array_all_null_tail) {
9312+
// An all-NULL tail chunk of an ARRAY subcolumn must be flushed through the array writer (which
9313+
// updates offsets, nulls and the inverted index together), not through append_nulls.
9314+
init_extracted_variant_schema(11407);
9315+
const std::vector<std::string> jsons {"[1,2]", "[\"a\",\"b\"]", "null", "null"};
9316+
ColumnPtr source;
9317+
DataTypePtr source_type;
9318+
ASSERT_TRUE(create_variant_writer_source(VariantWriterInput::V2, jsons, 0, false, {}, &source,
9319+
&source_type)
9320+
.ok());
9321+
9322+
SegmentFooterPB ref_footer;
9323+
std::string ref_path;
9324+
ASSERT_TRUE(write_extracted_variant_segment(source, source_type, "subcolumn_array_ref",
9325+
&ref_footer, &ref_path)
9326+
.ok());
9327+
std::vector<std::string> ref_rows;
9328+
ASSERT_TRUE(read_extracted_variant_rows(ref_footer, ref_path, &ref_rows).ok());
9329+
9330+
struct ChunkGuard {
9331+
ChunkGuard() { VariantSubcolumnWriter::set_chunk_values_for_test(2); }
9332+
~ChunkGuard() { VariantSubcolumnWriter::set_chunk_values_for_test(0); }
9333+
} guard;
9334+
segment_v2::variant_writer_helpers::reset_array_all_null_chunks_for_test();
9335+
size_t max_chunks = 0;
9336+
SegmentFooterPB chunked_footer;
9337+
std::string chunked_path;
9338+
ASSERT_TRUE(write_extracted_variant_segment(
9339+
source, source_type, "subcolumn_array_chunked", &chunked_footer,
9340+
&chunked_path, 0, 1,
9341+
[&](ColumnWriter* writer) {
9342+
auto* subcolumn = dynamic_cast<VariantSubcolumnWriter*>(writer);
9343+
ASSERT_NE(subcolumn, nullptr);
9344+
max_chunks = std::max(max_chunks, subcolumn->num_chunks_for_test());
9345+
})
9346+
.ok());
9347+
EXPECT_GE(max_chunks, 2);
9348+
EXPECT_EQ(segment_v2::variant_writer_helpers::array_all_null_chunks_for_test(), 1u);
9349+
9350+
std::vector<std::string> chunked_rows;
9351+
ASSERT_TRUE(read_extracted_variant_rows(chunked_footer, chunked_path, &chunked_rows).ok());
9352+
EXPECT_EQ(chunked_rows, ref_rows);
9353+
ASSERT_EQ(chunked_rows.size(), 4);
9354+
EXPECT_EQ(chunked_rows[2], "NULL");
9355+
EXPECT_EQ(chunked_rows[3], "NULL");
9356+
}
9357+
92559358
} // namespace doris

0 commit comments

Comments
 (0)