Skip to content

Commit 5314b85

Browse files
committed
[test](variant) Cover chunked VariantSubcolumnWriter edge cases (CIR-22037)
1 parent 92db91d commit 5314b85

3 files changed

Lines changed: 331 additions & 5 deletions

File tree

‎be/src/storage/segment/column_writer.h‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -643,6 +643,14 @@ class VariantSubcolumnWriter : public ColumnWriter {
643643

644644
size_t get_non_null_size() const { return none_null_size; }
645645

646+
#ifdef BE_TEST
647+
// Force small chunk budgets so tests can exercise multi-chunk flushes cheaply.
648+
static void set_chunk_bytes_for_test(size_t bytes);
649+
static void set_chunk_values_for_test(size_t values);
650+
size_t num_chunks_for_test() const;
651+
size_t max_chunk_bytes_for_test() const;
652+
#endif
653+
646654
Status finalize();
647655

648656
private:

‎be/src/storage/segment/variant/variant_column_writer_impl.cpp‎

Lines changed: 40 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -548,6 +548,11 @@ uint64_t VariantColumnWriterImpl::get_total_data_pages_bytes(
548548
return _v2_writer->get_total_data_pages_bytes(getter);
549549
}
550550

551+
#ifdef BE_TEST
552+
static size_t g_chunk_bytes_for_test = 0;
553+
static size_t g_chunk_values_for_test = 0;
554+
#endif
555+
551556
VariantSubcolumnWriter::VariantSubcolumnWriter(const ColumnWriterOptions& opts,
552557
TabletColumnPtr column)
553558
: ColumnWriter(std::move(column), opts.meta->is_nullable(), opts.meta) {
@@ -628,7 +633,19 @@ Status VariantSubcolumnWriter::_append_v2(const VariantColumnData& column, size_
628633
RETURN_IF_ERROR(append_encoded(encoded_batch->read_view(), 0));
629634
}
630635
_chunk_rows = _num_rows + num_rows - _chunk_row_begin;
631-
if (_v2_builder->byte_size() >= kChunkBytes || _v2_builder->non_null_rows() >= kChunkValues) {
636+
const size_t chunk_bytes =
637+
#ifdef BE_TEST
638+
g_chunk_bytes_for_test != 0 ? g_chunk_bytes_for_test : kChunkBytes;
639+
#else
640+
kChunkBytes;
641+
#endif
642+
const size_t chunk_values =
643+
#ifdef BE_TEST
644+
g_chunk_values_for_test != 0 ? g_chunk_values_for_test : kChunkValues;
645+
#else
646+
kChunkValues;
647+
#endif
648+
if (_v2_builder->byte_size() >= chunk_bytes || _v2_builder->non_null_rows() >= chunk_values) {
632649
RETURN_IF_ERROR(_seal_chunk());
633650
}
634651
return Status::OK();
@@ -669,6 +686,28 @@ Status VariantSubcolumnWriter::append_data(const uint8_t** ptr, size_t num_rows)
669686
return _append(nullptr, ptr, num_rows);
670687
}
671688

689+
#ifdef BE_TEST
690+
void VariantSubcolumnWriter::set_chunk_bytes_for_test(size_t bytes) {
691+
g_chunk_bytes_for_test = bytes;
692+
}
693+
694+
void VariantSubcolumnWriter::set_chunk_values_for_test(size_t values) {
695+
g_chunk_values_for_test = values;
696+
}
697+
698+
size_t VariantSubcolumnWriter::num_chunks_for_test() const {
699+
return _chunks.size() + (_v2_builder != nullptr ? 1 : 0);
700+
}
701+
702+
size_t VariantSubcolumnWriter::max_chunk_bytes_for_test() const {
703+
size_t max = _v2_builder != nullptr ? _v2_builder->byte_size() : 0;
704+
for (const auto& chunk : _chunks) {
705+
max = std::max(max, chunk.builder->byte_size());
706+
}
707+
return max;
708+
}
709+
#endif
710+
672711
uint64_t VariantSubcolumnWriter::estimate_buffer_size() {
673712
if (is_finalized()) {
674713
return _writer ? _writer->estimate_buffer_size() : 0;

‎be/test/storage/variant/variant_column_writer_reader_test.cpp‎

Lines changed: 283 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1327,6 +1327,25 @@ static Status create_variant_root_reader(const SegmentFooterPB& footer,
13271327

13281328
class VariantColumnWriterReaderTest : public testing::Test {
13291329
public:
1330+
// Parent VARIANT column with an extracted VARIANT subcolumn `v.payload`, i.e. the layout
1331+
// VariantSubcolumnWriter is responsible for.
1332+
void init_extracted_variant_schema(int64_t tablet_id) {
1333+
TabletSchemaPB schema_pb;
1334+
schema_pb.set_keys_type(KeysType::DUP_KEYS);
1335+
construct_column(schema_pb.add_column(), 1, "VARIANT", "v", 1);
1336+
_tablet_schema = std::make_shared<TabletSchema>();
1337+
_tablet_schema->init_from_pb(schema_pb);
1338+
const TabletColumn& parent_column = _tablet_schema->column_by_uid(1);
1339+
TabletColumn extracted_column;
1340+
extracted_column.set_name(parent_column.name_lower_case() + ".payload");
1341+
extracted_column.set_type(FieldType::OLAP_FIELD_TYPE_VARIANT);
1342+
extracted_column.set_parent_unique_id(parent_column.unique_id());
1343+
extracted_column.set_path_info(PathInData(parent_column.name_lower_case() + ".payload"));
1344+
extracted_column.set_is_nullable(true);
1345+
_tablet_schema->append_column(extracted_column);
1346+
init_tablet_from_current_schema(tablet_id);
1347+
}
1348+
13301349
void SetUp() override {
13311350
// absolute dir
13321351
char buffer[MAX_PATH_LEN];
@@ -1721,9 +1740,10 @@ class VariantColumnWriterReaderTest : public testing::Test {
17211740
return Status::OK();
17221741
}
17231742

1724-
Status write_extracted_variant_segment(const ColumnPtr& source, const DataTypePtr& source_type,
1725-
std::string_view rowset_id, SegmentFooterPB* footer,
1726-
std::string* file_path, size_t first_batch_rows = 0) {
1743+
Status write_extracted_variant_segment(
1744+
const ColumnPtr& source, const DataTypePtr& source_type, std::string_view rowset_id,
1745+
SegmentFooterPB* footer, std::string* file_path, size_t first_batch_rows = 0,
1746+
size_t batch_rows = 0, const std::function<void(ColumnWriter*)>& after_batch = {}) {
17271747
DORIS_CHECK(source.get() != nullptr);
17281748
DORIS_CHECK(source_type != nullptr);
17291749
DORIS_CHECK(footer != nullptr);
@@ -1768,11 +1788,28 @@ class VariantColumnWriterReaderTest : public testing::Test {
17681788
converter.clear_source_content(0);
17691789
return Status::OK();
17701790
};
1771-
if (first_batch_rows > 0 && first_batch_rows < source->size()) {
1791+
if (batch_rows > 0) {
1792+
for (size_t row_pos = 0; row_pos < source->size(); row_pos += batch_rows) {
1793+
const size_t rows = std::min(batch_rows, source->size() - row_pos);
1794+
RETURN_IF_ERROR(append_batch(row_pos, rows));
1795+
if (after_batch) {
1796+
after_batch(writer.get());
1797+
}
1798+
}
1799+
} else if (first_batch_rows > 0 && first_batch_rows < source->size()) {
17721800
RETURN_IF_ERROR(append_batch(0, first_batch_rows));
1801+
if (after_batch) {
1802+
after_batch(writer.get());
1803+
}
17731804
RETURN_IF_ERROR(append_batch(first_batch_rows, source->size() - first_batch_rows));
1805+
if (after_batch) {
1806+
after_batch(writer.get());
1807+
}
17741808
} else {
17751809
RETURN_IF_ERROR(append_batch(0, source->size()));
1810+
if (after_batch) {
1811+
after_batch(writer.get());
1812+
}
17761813
}
17771814

17781815
RETURN_IF_ERROR(writer->finish());
@@ -8973,4 +9010,246 @@ TEST_F(VariantColumnWriterReaderTest, legacy_v1_segment_compaction_preserves_ful
89739010
EXPECT_EQ(actual_rows, expected_rows);
89749011
}
89759012

9013+
TEST_F(VariantColumnWriterReaderTest, test_variant_subcolumn_writer_chunks_large_values) {
9014+
// A large VARIANT subcolumn must be flushed in bounded chunks: a single segment-sized
9015+
// ColumnString is what overflows ColumnStr<uint32_t> at 4 GiB. Compare the chunked output with
9016+
// the single-chunk reference and check the observed chunk budget.
9017+
init_extracted_variant_schema(11401);
9018+
constexpr size_t kRows = 3000;
9019+
constexpr size_t kBatchRows = 300;
9020+
const std::string value(1024, 'x');
9021+
const std::string json = "\"" + value + "\"";
9022+
std::vector<std::string> jsons(kRows, json);
9023+
ColumnPtr source;
9024+
DataTypePtr source_type;
9025+
ASSERT_TRUE(create_variant_writer_source(VariantWriterInput::V2, jsons, 0, false, {}, &source,
9026+
&source_type)
9027+
.ok());
9028+
9029+
SegmentFooterPB ref_footer;
9030+
std::string ref_path;
9031+
ASSERT_TRUE(write_extracted_variant_segment(source, source_type, "subcolumn_chunk_ref",
9032+
&ref_footer, &ref_path)
9033+
.ok());
9034+
std::vector<std::string> ref_rows;
9035+
ASSERT_TRUE(read_extracted_variant_rows(ref_footer, ref_path, &ref_rows).ok());
9036+
ASSERT_EQ(ref_rows.size(), kRows);
9037+
9038+
struct ChunkGuard {
9039+
ChunkGuard() { VariantSubcolumnWriter::set_chunk_values_for_test(250); }
9040+
~ChunkGuard() { VariantSubcolumnWriter::set_chunk_values_for_test(0); }
9041+
} guard;
9042+
size_t max_chunks = 0;
9043+
size_t max_chunk_bytes = 0;
9044+
SegmentFooterPB chunked_footer;
9045+
std::string chunked_path;
9046+
ASSERT_TRUE(write_extracted_variant_segment(
9047+
source, source_type, "subcolumn_chunk_values", &chunked_footer,
9048+
&chunked_path, 0, kBatchRows,
9049+
[&](ColumnWriter* writer) {
9050+
auto* subcolumn = dynamic_cast<VariantSubcolumnWriter*>(writer);
9051+
ASSERT_NE(subcolumn, nullptr);
9052+
max_chunks = std::max(max_chunks, subcolumn->num_chunks_for_test());
9053+
max_chunk_bytes = std::max(max_chunk_bytes,
9054+
subcolumn->max_chunk_bytes_for_test());
9055+
})
9056+
.ok());
9057+
EXPECT_GE(max_chunks, 2);
9058+
// A chunk holds at most the configured budget plus the batch that crossed it.
9059+
EXPECT_LE(max_chunk_bytes, 250 * 1024 + kBatchRows * 1024);
9060+
9061+
std::vector<std::string> chunked_rows;
9062+
ASSERT_TRUE(read_extracted_variant_rows(chunked_footer, chunked_path, &chunked_rows).ok());
9063+
EXPECT_EQ(chunked_rows, ref_rows);
9064+
EXPECT_EQ(chunked_rows.front(), ref_rows.front());
9065+
EXPECT_EQ(chunked_rows.back(), ref_rows.back());
9066+
}
9067+
9068+
TEST_F(VariantColumnWriterReaderTest, test_variant_subcolumn_writer_chunks_type_promotion) {
9069+
// Values that widen the inferred type across a chunk boundary (int -> string -> JSONB) must
9070+
// still produce the same column as the single-chunk path.
9071+
init_extracted_variant_schema(11402);
9072+
constexpr size_t kHalf = 2000;
9073+
std::vector<std::string> jsons;
9074+
jsons.reserve(2 * kHalf);
9075+
for (size_t i = 0; i < kHalf; ++i) {
9076+
jsons.push_back(std::to_string(i));
9077+
}
9078+
for (size_t i = 0; i < kHalf; ++i) {
9079+
jsons.push_back("\"s" + std::to_string(i) + "\"");
9080+
}
9081+
ColumnPtr source;
9082+
DataTypePtr source_type;
9083+
ASSERT_TRUE(create_variant_writer_source(VariantWriterInput::V2, jsons, 0, false, {}, &source,
9084+
&source_type)
9085+
.ok());
9086+
9087+
SegmentFooterPB ref_footer;
9088+
std::string ref_path;
9089+
ASSERT_TRUE(write_extracted_variant_segment(source, source_type, "subcolumn_promote_ref",
9090+
&ref_footer, &ref_path)
9091+
.ok());
9092+
std::vector<std::string> ref_rows;
9093+
ASSERT_TRUE(read_extracted_variant_rows(ref_footer, ref_path, &ref_rows).ok());
9094+
9095+
struct ChunkGuard {
9096+
ChunkGuard() { VariantSubcolumnWriter::set_chunk_values_for_test(500); }
9097+
~ChunkGuard() { VariantSubcolumnWriter::set_chunk_values_for_test(0); }
9098+
} guard;
9099+
size_t max_chunks = 0;
9100+
SegmentFooterPB chunked_footer;
9101+
std::string chunked_path;
9102+
ASSERT_TRUE(write_extracted_variant_segment(
9103+
source, source_type, "subcolumn_promote_chunked", &chunked_footer,
9104+
&chunked_path, 0, 300,
9105+
[&](ColumnWriter* writer) {
9106+
auto* subcolumn = dynamic_cast<VariantSubcolumnWriter*>(writer);
9107+
ASSERT_NE(subcolumn, nullptr);
9108+
max_chunks = std::max(max_chunks, subcolumn->num_chunks_for_test());
9109+
})
9110+
.ok());
9111+
EXPECT_GE(max_chunks, 2);
9112+
9113+
std::vector<std::string> chunked_rows;
9114+
ASSERT_TRUE(read_extracted_variant_rows(chunked_footer, chunked_path, &chunked_rows).ok());
9115+
EXPECT_EQ(chunked_rows, ref_rows);
9116+
}
9117+
9118+
TEST_F(VariantColumnWriterReaderTest, test_variant_subcolumn_writer_chunks_sparse_rows) {
9119+
// Rows without the subcolumn become nulls. When chunks split a gap the dense materialization
9120+
// must still place every value on its logical row.
9121+
init_extracted_variant_schema(11403);
9122+
constexpr size_t kRows = 3000;
9123+
std::vector<std::string> jsons(kRows, "\"v\"");
9124+
std::vector<UInt8> outer_nulls(kRows, 0);
9125+
for (size_t row = 0; row < kRows; row += 3) {
9126+
outer_nulls[row] = 1;
9127+
}
9128+
ColumnPtr source;
9129+
DataTypePtr source_type;
9130+
ASSERT_TRUE(create_variant_writer_source(VariantWriterInput::V2, jsons, 0, false, outer_nulls,
9131+
&source, &source_type)
9132+
.ok());
9133+
9134+
SegmentFooterPB ref_footer;
9135+
std::string ref_path;
9136+
ASSERT_TRUE(write_extracted_variant_segment(source, source_type, "subcolumn_sparse_ref",
9137+
&ref_footer, &ref_path)
9138+
.ok());
9139+
std::vector<std::string> ref_rows;
9140+
ASSERT_TRUE(read_extracted_variant_rows(ref_footer, ref_path, &ref_rows).ok());
9141+
9142+
struct ChunkGuard {
9143+
ChunkGuard() { VariantSubcolumnWriter::set_chunk_values_for_test(200); }
9144+
~ChunkGuard() { VariantSubcolumnWriter::set_chunk_values_for_test(0); }
9145+
} guard;
9146+
size_t max_chunks = 0;
9147+
SegmentFooterPB chunked_footer;
9148+
std::string chunked_path;
9149+
ASSERT_TRUE(write_extracted_variant_segment(
9150+
source, source_type, "subcolumn_sparse_chunked", &chunked_footer,
9151+
&chunked_path, 0, 300,
9152+
[&](ColumnWriter* writer) {
9153+
auto* subcolumn = dynamic_cast<VariantSubcolumnWriter*>(writer);
9154+
ASSERT_NE(subcolumn, nullptr);
9155+
max_chunks = std::max(max_chunks, subcolumn->num_chunks_for_test());
9156+
})
9157+
.ok());
9158+
EXPECT_GE(max_chunks, 2);
9159+
9160+
std::vector<std::string> chunked_rows;
9161+
ASSERT_TRUE(read_extracted_variant_rows(chunked_footer, chunked_path, &chunked_rows).ok());
9162+
EXPECT_EQ(chunked_rows, ref_rows);
9163+
ASSERT_FALSE(chunked_rows.empty());
9164+
EXPECT_EQ(chunked_rows[0], "NULL");
9165+
EXPECT_EQ(chunked_rows[1], ref_rows[1]);
9166+
}
9167+
9168+
TEST_F(VariantColumnWriterReaderTest, test_variant_subcolumn_writer_chunks_all_null) {
9169+
// A subcolumn whose every row is null must still append the right number of nulls per chunk.
9170+
init_extracted_variant_schema(11404);
9171+
constexpr size_t kRows = 2000;
9172+
std::vector<std::string> jsons(kRows, "\"v\"");
9173+
std::vector<UInt8> outer_nulls(kRows, 1);
9174+
ColumnPtr source;
9175+
DataTypePtr source_type;
9176+
ASSERT_TRUE(create_variant_writer_source(VariantWriterInput::V2, jsons, 0, false, outer_nulls,
9177+
&source, &source_type)
9178+
.ok());
9179+
9180+
SegmentFooterPB ref_footer;
9181+
std::string ref_path;
9182+
ASSERT_TRUE(write_extracted_variant_segment(source, source_type, "subcolumn_null_ref",
9183+
&ref_footer, &ref_path)
9184+
.ok());
9185+
std::vector<std::string> ref_rows;
9186+
ASSERT_TRUE(read_extracted_variant_rows(ref_footer, ref_path, &ref_rows).ok());
9187+
9188+
struct ChunkGuard {
9189+
ChunkGuard() { VariantSubcolumnWriter::set_chunk_values_for_test(100); }
9190+
~ChunkGuard() { VariantSubcolumnWriter::set_chunk_values_for_test(0); }
9191+
} guard;
9192+
SegmentFooterPB chunked_footer;
9193+
std::string chunked_path;
9194+
ASSERT_TRUE(write_extracted_variant_segment(source, source_type, "subcolumn_null_chunked",
9195+
&chunked_footer, &chunked_path, 0, 300)
9196+
.ok());
9197+
std::vector<std::string> chunked_rows;
9198+
ASSERT_TRUE(read_extracted_variant_rows(chunked_footer, chunked_path, &chunked_rows).ok());
9199+
EXPECT_EQ(chunked_rows, ref_rows);
9200+
for (const auto& row : chunked_rows) {
9201+
EXPECT_EQ(row, "NULL");
9202+
}
9203+
}
9204+
9205+
TEST_F(VariantColumnWriterReaderTest, test_variant_subcolumn_writer_chunks_by_bytes) {
9206+
// The byte budget (kChunkBytes) seals a chunk even when the value count is small, e.g. few very
9207+
// wide strings. The chunk must still be flushed independently and read back unchanged.
9208+
init_extracted_variant_schema(11405);
9209+
constexpr size_t kRows = 3000;
9210+
constexpr size_t kBatchRows = 200;
9211+
const std::string value(1024, 'y');
9212+
const std::string json = "\"" + value + "\"";
9213+
std::vector<std::string> jsons(kRows, json);
9214+
ColumnPtr source;
9215+
DataTypePtr source_type;
9216+
ASSERT_TRUE(create_variant_writer_source(VariantWriterInput::V2, jsons, 0, false, {}, &source,
9217+
&source_type)
9218+
.ok());
9219+
9220+
SegmentFooterPB ref_footer;
9221+
std::string ref_path;
9222+
ASSERT_TRUE(write_extracted_variant_segment(source, source_type, "subcolumn_bytes_ref",
9223+
&ref_footer, &ref_path)
9224+
.ok());
9225+
std::vector<std::string> ref_rows;
9226+
ASSERT_TRUE(read_extracted_variant_rows(ref_footer, ref_path, &ref_rows).ok());
9227+
9228+
struct ChunkGuard {
9229+
ChunkGuard() { VariantSubcolumnWriter::set_chunk_bytes_for_test(256 * 1024); }
9230+
~ChunkGuard() { VariantSubcolumnWriter::set_chunk_bytes_for_test(0); }
9231+
} guard;
9232+
size_t max_chunks = 0;
9233+
size_t max_chunk_bytes = 0;
9234+
SegmentFooterPB chunked_footer;
9235+
std::string chunked_path;
9236+
ASSERT_TRUE(write_extracted_variant_segment(
9237+
source, source_type, "subcolumn_bytes_chunked", &chunked_footer,
9238+
&chunked_path, 0, kBatchRows,
9239+
[&](ColumnWriter* writer) {
9240+
auto* subcolumn = dynamic_cast<VariantSubcolumnWriter*>(writer);
9241+
ASSERT_NE(subcolumn, nullptr);
9242+
max_chunks = std::max(max_chunks, subcolumn->num_chunks_for_test());
9243+
max_chunk_bytes = std::max(max_chunk_bytes,
9244+
subcolumn->max_chunk_bytes_for_test());
9245+
})
9246+
.ok());
9247+
EXPECT_GE(max_chunks, 2);
9248+
EXPECT_LE(max_chunk_bytes, 256 * 1024 + kBatchRows * 1024);
9249+
9250+
std::vector<std::string> chunked_rows;
9251+
ASSERT_TRUE(read_extracted_variant_rows(chunked_footer, chunked_path, &chunked_rows).ok());
9252+
EXPECT_EQ(chunked_rows, ref_rows);
9253+
}
9254+
89769255
} // namespace doris

0 commit comments

Comments
 (0)