Skip to content

Commit cf1db49

Browse files
committed
[test](variant) Reproduce decimal inference across writer chunks
### What problem does this PR solve? Related PR: apache#69221 Problem Summary: Chunked extracted VARIANT writes can discard valid Decimal values or reduce decimal-array scale. Add native Decimal writer/readback cases comparing chunked and uninterrupted inference, including a later Double and values retained in the active chunk, plus a direct nullable-array type-merge check. ### Release note None ### Check List (For Author) - Test: ASAN BE unit tests: all three added tests fail on the unmodified implementation (red); clang-format 16 and build hygiene pass - Behavior changed: No (tests only) - Does this need documentation: No
1 parent 720f7f7 commit cf1db49

1 file changed

Lines changed: 133 additions & 0 deletions

File tree

‎be/test/storage/variant/variant_column_writer_reader_test.cpp‎

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9355,4 +9355,137 @@ TEST_F(VariantColumnWriterReaderTest, test_variant_subcolumn_writer_chunks_array
93559355
EXPECT_EQ(chunked_rows[3], "NULL");
93569356
}
93579357

9358+
TEST(VariantPathBuilderTest, MergeNullableDecimalArraysPreservesScale) {
9359+
VariantBatchBuilder value_builder;
9360+
for (size_t index = 0; index < 2; ++index) {
9361+
auto row = value_builder.begin_row();
9362+
auto array = row.start_array();
9363+
row.add_decimal(index == 0 ? 123 : 12345, index == 0 ? 2 : 4, 16);
9364+
array.finish();
9365+
row.finish();
9366+
}
9367+
const auto values = value_builder.finish_batch();
9368+
segment_v2::VariantPathBuilder first(PathInData("metric"));
9369+
segment_v2::VariantPathBuilder second(PathInData("metric"));
9370+
ASSERT_TRUE(first.append(values.value_at(0), 0).ok());
9371+
ASSERT_TRUE(second.append(values.value_at(1), 0).ok());
9372+
for (bool reverse : {false, true}) {
9373+
const DataTypePtr common = segment_v2::variant_path_least_common_type(
9374+
reverse ? second.type() : first.type(), reverse ? first.type() : second.type());
9375+
EXPECT_TRUE(common->is_nullable());
9376+
const auto& array = assert_cast<const DataTypeArray&>(*remove_nullable(common));
9377+
EXPECT_EQ(array.get_nested_type()->get_scale(), 4);
9378+
}
9379+
}
9380+
9381+
TEST_F(VariantColumnWriterReaderTest, test_variant_subcolumn_writer_chunks_decimal_overflow) {
9382+
init_extracted_variant_schema(11408);
9383+
for (bool array_values : {false, true}) {
9384+
for (bool double_tail : {false, true}) {
9385+
SCOPED_TRACE(testing::Message()
9386+
<< "array=" << array_values << ", double_tail=" << double_tail);
9387+
VariantBatchBuilder value_builder;
9388+
for (size_t index = 0; index < (double_tail ? 5 : 4); ++index) {
9389+
auto row = value_builder.begin_row();
9390+
std::optional<VariantBatchBuilder::Row::ArrayScope> array;
9391+
if (array_values) {
9392+
array.emplace(row.start_array());
9393+
}
9394+
if (index == 1) {
9395+
row.add_decimal(static_cast<__int128>(VARIANT_DECIMAL16_MAX), 1, 16);
9396+
} else if (index == 3) {
9397+
row.add_decimal(1, 38, 16);
9398+
} else if (index == 4) {
9399+
row.add_double(2.5);
9400+
} else {
9401+
row.add_decimal(1, 1, 16);
9402+
}
9403+
if (array_values) {
9404+
array->finish();
9405+
}
9406+
row.finish();
9407+
}
9408+
auto source = ColumnVariantV2::create();
9409+
source->insert_encoded_batch(value_builder.finish_batch());
9410+
const auto source_type = std::make_shared<DataTypeVariantV2>(0, false);
9411+
const auto suffix = fmt::format("{}_{}", array_values, double_tail);
9412+
SegmentFooterPB ref_footer;
9413+
std::string ref_path;
9414+
ASSERT_TRUE(write_extracted_variant_segment(source->get_ptr(), source_type,
9415+
"decimal_overflow_ref_" + suffix,
9416+
&ref_footer, &ref_path)
9417+
.ok());
9418+
std::vector<std::string> ref_rows;
9419+
ASSERT_TRUE(read_extracted_variant_rows(ref_footer, ref_path, &ref_rows).ok());
9420+
ASSERT_EQ(static_cast<FieldType>(ref_footer.columns(0).type()),
9421+
FieldType::OLAP_FIELD_TYPE_JSONB);
9422+
9423+
struct ChunkGuard {
9424+
explicit ChunkGuard(size_t values) {
9425+
VariantSubcolumnWriter::set_chunk_values_for_test(values);
9426+
}
9427+
~ChunkGuard() { VariantSubcolumnWriter::set_chunk_values_for_test(0); }
9428+
} guard(double_tail ? 2 : 1);
9429+
SegmentFooterPB chunked_footer;
9430+
std::string chunked_path;
9431+
ASSERT_TRUE(write_extracted_variant_segment(source->get_ptr(), source_type,
9432+
"decimal_overflow_chunks_" + suffix,
9433+
&chunked_footer, &chunked_path, 0, 1)
9434+
.ok());
9435+
std::vector<std::string> chunked_rows;
9436+
ASSERT_TRUE(
9437+
read_extracted_variant_rows(chunked_footer, chunked_path, &chunked_rows).ok());
9438+
// The scale-38 value cannot widen the earlier large value losslessly. With two rows
9439+
// per chunk, fallback must also preserve the preceding 0.1 in the active chunk.
9440+
EXPECT_EQ(chunked_rows, ref_rows);
9441+
EXPECT_EQ(static_cast<FieldType>(chunked_footer.columns(0).type()),
9442+
FieldType::OLAP_FIELD_TYPE_JSONB);
9443+
}
9444+
}
9445+
}
9446+
9447+
TEST_F(VariantColumnWriterReaderTest, test_variant_subcolumn_writer_chunks_decimal_array_scales) {
9448+
init_extracted_variant_schema(11409);
9449+
for (bool reverse : {false, true}) {
9450+
SCOPED_TRACE(testing::Message() << "reverse=" << reverse);
9451+
VariantBatchBuilder value_builder;
9452+
for (size_t index = 0; index < 2; ++index) {
9453+
auto row = value_builder.begin_row();
9454+
auto array = row.start_array();
9455+
const bool high_scale = (index == 1) != reverse;
9456+
row.add_decimal(high_scale ? 12345 : 123, high_scale ? 4 : 2, 16);
9457+
array.finish();
9458+
row.finish();
9459+
}
9460+
auto source = ColumnVariantV2::create();
9461+
source->insert_encoded_batch(value_builder.finish_batch());
9462+
const auto source_type = std::make_shared<DataTypeVariantV2>(0, false);
9463+
const auto suffix = std::to_string(reverse);
9464+
SegmentFooterPB ref_footer;
9465+
std::string ref_path;
9466+
ASSERT_TRUE(write_extracted_variant_segment(source->get_ptr(), source_type,
9467+
"decimal_array_ref_" + suffix, &ref_footer,
9468+
&ref_path)
9469+
.ok());
9470+
std::vector<std::string> ref_rows;
9471+
ASSERT_TRUE(read_extracted_variant_rows(ref_footer, ref_path, &ref_rows).ok());
9472+
ASSERT_EQ(ref_footer.columns(0).children_columns(0).frac(), 4);
9473+
9474+
struct ChunkGuard {
9475+
ChunkGuard() { VariantSubcolumnWriter::set_chunk_values_for_test(1); }
9476+
~ChunkGuard() { VariantSubcolumnWriter::set_chunk_values_for_test(0); }
9477+
} guard;
9478+
SegmentFooterPB chunked_footer;
9479+
std::string chunked_path;
9480+
ASSERT_TRUE(write_extracted_variant_segment(source->get_ptr(), source_type,
9481+
"decimal_array_chunks_" + suffix,
9482+
&chunked_footer, &chunked_path, 0, 1)
9483+
.ok());
9484+
std::vector<std::string> chunked_rows;
9485+
ASSERT_TRUE(read_extracted_variant_rows(chunked_footer, chunked_path, &chunked_rows).ok());
9486+
EXPECT_EQ(chunked_rows, ref_rows);
9487+
EXPECT_EQ(chunked_footer.columns(0).children_columns(0).frac(), 4);
9488+
}
9489+
}
9490+
93589491
} // namespace doris

0 commit comments

Comments
 (0)