Skip to content

Commit fab1e0c

Browse files
committed
[fix](variant) Preserve inferred values across writer chunks
### What problem does this PR solve? Related PR: apache#69221 Problem Summary: Extracted VARIANT chunks inherit a type without validating it against earlier values. Decimal widening can therefore filter valid rows as cast NULLs, and nullable-array type merging can reduce element scale. Resolve inferred promotions against retained chunks before committing any cast, preserving whole-path JSONB fallback and the active chunk's original Decimal representation. Unwrap nullable types before array/Decimal dispatch. Keep explicit typed-storage cast-null filtering unchanged. ### Release note Preserve VARIANT Decimal values and decimal-array precision during chunked writes. ### Check List (For Author) - Test: ASAN BE unit tests: three new tests fail before the fix and pass afterward; related writer, path-builder, shredder and compatibility tests: 103 passed, 2 skipped because NestedGroup write support is unavailable. Local UT uses GLIBC_COMPATIBILITY=OFF to avoid duplicate toolchain compatibility symbols. clang-format 16, build hygiene and clang-tidy on changed lines passed. - Behavior changed: Yes, inferred cross-chunk conversions preserve valid values and Decimal array scale - Does this need documentation: No
1 parent cf1db49 commit fab1e0c

4 files changed

Lines changed: 98 additions & 23 deletions

File tree

‎be/src/storage/segment/column_writer.h‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -659,6 +659,7 @@ class VariantSubcolumnWriter : public ColumnWriter {
659659
std::span<const uint8_t> outer_nulls);
660660
Status _ensure_input_format(const VariantColumnData& column);
661661
Status _initialize_v2_builder();
662+
Status _promote_chunks(DataTypePtr* inferred_type);
662663
Status _seal_chunk();
663664
// Bounded value chunks; each chunk is finalized and flushed independently so the writer never
664665
// holds a segment-sized ColumnString.
@@ -678,8 +679,8 @@ class VariantSubcolumnWriter : public ColumnWriter {
678679
std::vector<ValueChunk> _chunks;
679680
size_t _chunk_row_begin = 0;
680681
size_t _chunk_rows = 0;
681-
// Type accumulated by the sealed chunks. A new chunk starts from it so appends follow the
682-
// exact same inference and JSONB-fallback sequence as one uninterrupted builder.
682+
// Type carried into the next chunk. Inferred promotions also check and update sealed chunks
683+
// so fallback follows the same sequence as one uninterrupted builder.
683684
DataTypePtr _running_type;
684685
size_t _num_rows = 0;
685686
ColumnWriterOptions _opts;

‎be/src/storage/segment/variant/v2/variant_path_builder.cpp‎

Lines changed: 53 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -371,6 +371,11 @@ DataTypePtr path_least_common_type(const DataTypePtr& left, const DataTypePtr& r
371371
if (left.get() == right.get() || left->equals(*right)) {
372372
return left;
373373
}
374+
// Chunk builders expose Nullable(Array(...)). Unwrap before dispatching so decimal array
375+
// elements use the scale-aware rules below instead of the generic first-primitive match.
376+
if (left->is_nullable() || right->is_nullable()) {
377+
return make_nullable(path_least_common_type(remove_nullable(left), remove_nullable(right)));
378+
}
374379
const auto* left_array = typeid_cast<const DataTypeArray*>(left.get());
375380
const auto* right_array = typeid_cast<const DataTypeArray*>(right.get());
376381
if (left_array != nullptr || right_array != nullptr) {
@@ -1082,39 +1087,54 @@ struct VariantPathBuilder::Impl {
10821087
return Status::OK();
10831088
}
10841089

1085-
Status promote(const DataTypePtr& target_type, bool filter_cast_nulls) {
1086-
DataTypePtr target = remove_nullable(target_type);
1087-
if (target->equals(*type)) {
1088-
return Status::OK();
1089-
}
1090-
ColumnPtr promoted;
1090+
Status cast_to_type(const DataTypePtr& target, ColumnPtr* promoted) const {
10911091
if (target->get_primitive_type() == TYPE_STRING &&
10921092
type->get_primitive_type() == TYPE_ARRAY) {
10931093
RETURN_IF_ERROR(stringify_complex_column(
1094-
type, assert_cast<const ColumnNullable&>(*column), &promoted));
1094+
type, assert_cast<const ColumnNullable&>(*column), promoted));
10951095
} else if (type->get_primitive_type() == TYPE_ARRAY &&
10961096
target->get_primitive_type() == TYPE_ARRAY &&
10971097
variant_path_type_contains_nothing(type)) {
10981098
MutableColumnPtr materialized;
10991099
RETURN_IF_ERROR(replace_array_nothing(nullable_type, *column, make_nullable(target),
11001100
&materialized));
1101-
promoted = std::move(materialized);
1101+
*promoted = std::move(materialized);
11021102
} else {
11031103
RETURN_IF_ERROR(
11041104
variant_util::cast_column({column->get_ptr(), nullable_type, path.get_path()},
1105-
make_nullable(target), &promoted));
1105+
make_nullable(target), promoted));
11061106
}
1107+
return Status::OK();
1108+
}
1109+
1110+
Status promote(const DataTypePtr& target_type, bool filter_cast_nulls,
1111+
const InferredPromotion& on_promotion = {}) {
1112+
DataTypePtr target = remove_nullable(target_type);
1113+
if (target->equals(*type)) {
1114+
return Status::OK();
1115+
}
1116+
ColumnPtr promoted;
1117+
RETURN_IF_ERROR(cast_to_type(target, &promoted));
11071118
// Inferred widening is only a storage representation choice. If CAST loses a valid value
11081119
// at any array depth, preserve the whole path as JSONB instead. Forced typed-path
11091120
// conversion intentionally retains its existing cast-null filtering below.
11101121
if (!filter_cast_nulls && target->get_primitive_type() != TYPE_JSONB &&
11111122
cast_introduced_null(*column, *promoted)) {
11121123
target = jsonb_type();
1113-
RETURN_IF_ERROR(
1114-
variant_util::cast_column({column->get_ptr(), nullable_type, path.get_path()},
1115-
make_nullable(target), &promoted));
1124+
RETURN_IF_ERROR(cast_to_type(target, &promoted));
11161125
DORIS_CHECK(!assert_cast<const ColumnNullable&>(*promoted).has_null());
11171126
}
1127+
if (on_promotion) {
1128+
const DataTypePtr proposed = target;
1129+
RETURN_IF_ERROR(on_promotion(&target));
1130+
target = remove_nullable(target);
1131+
if (!target->equals(*proposed)) {
1132+
// For example, the active chunk's 0.1 fits scale 38, but a sealed large Decimal
1133+
// does not. Recast the original active values to JSONB, preserving 0.1's scale.
1134+
DORIS_CHECK_EQ(target->get_primitive_type(), TYPE_JSONB);
1135+
RETURN_IF_ERROR(cast_to_type(target, &promoted));
1136+
}
1137+
}
11181138
const auto& nullable = assert_cast<const ColumnNullable&>(*promoted);
11191139
if (nullable.has_null()) {
11201140
DORIS_CHECK(filter_cast_nulls);
@@ -1164,7 +1184,8 @@ VariantPathBuilder::~VariantPathBuilder() = default;
11641184
VariantPathBuilder::VariantPathBuilder(VariantPathBuilder&&) noexcept = default;
11651185
VariantPathBuilder& VariantPathBuilder::operator=(VariantPathBuilder&&) noexcept = default;
11661186

1167-
Status VariantPathBuilder::append(VariantRef value, size_t row) {
1187+
Status VariantPathBuilder::append(VariantRef value, size_t row,
1188+
const InferredPromotion& on_promotion) {
11681189
try {
11691190
const ScalarPhysical physical = scalar_physical(value);
11701191
if (physical.kind == ScalarPhysicalKind::NULL_VALUE) {
@@ -1193,14 +1214,14 @@ Status VariantPathBuilder::append(VariantRef value, size_t row) {
11931214
DataTypePtr common_type = path_least_common_type(_impl->type, inferred_type);
11941215
if (_impl->type.get() != common_type.get() &&
11951216
!_impl->type->equals(*common_type)) {
1196-
RETURN_IF_ERROR(_impl->promote(common_type, false));
1217+
RETURN_IF_ERROR(_impl->promote(common_type, false, on_promotion));
11971218
}
11981219
}
11991220
}
12001221
}
12011222
const bool is_array = !stable_scalar && value_kind(value) == ValueKind::ARRAY;
12021223
if (is_array && !value_is_representable(value, _impl->type)) {
1203-
RETURN_IF_ERROR(_impl->promote(jsonb_type(), false));
1224+
RETURN_IF_ERROR(_impl->promote(jsonb_type(), false, on_promotion));
12041225
}
12051226

12061227
try {
@@ -1211,7 +1232,7 @@ Status VariantPathBuilder::append(VariantRef value, size_t row) {
12111232
if (is_array || _impl->type->get_primitive_type() == TYPE_JSONB) {
12121233
throw;
12131234
}
1214-
RETURN_IF_ERROR(_impl->promote(jsonb_type(), false));
1235+
RETURN_IF_ERROR(_impl->promote(jsonb_type(), false, on_promotion));
12151236
auto& nullable = assert_cast<ColumnNullable&>(*_impl->column);
12161237
append_value(value, _impl->type, &nullable.get_nested_column());
12171238
nullable.get_null_map_data().push_back(0);
@@ -1236,11 +1257,25 @@ Status VariantPathBuilder::complete_rows(size_t rows) {
12361257
return Status::OK();
12371258
}
12381259

1239-
Status VariantPathBuilder::convert_to(const DataTypePtr& storage_type) {
1260+
Status VariantPathBuilder::convert_to(const DataTypePtr& storage_type, bool filter_cast_nulls) {
12401261
if (!_impl->column) {
12411262
return _impl->initialize(storage_type);
12421263
}
1243-
return _impl->promote(storage_type, true);
1264+
return _impl->promote(storage_type, filter_cast_nulls);
1265+
}
1266+
1267+
Status VariantPathBuilder::check_inferred_promotion(const DataTypePtr& target_type,
1268+
bool* needs_jsonb) const {
1269+
const DataTypePtr target = remove_nullable(target_type);
1270+
*needs_jsonb = false;
1271+
if (!_impl->column || target->equals(*_impl->type) ||
1272+
target->get_primitive_type() == TYPE_JSONB) {
1273+
return Status::OK();
1274+
}
1275+
ColumnPtr promoted;
1276+
RETURN_IF_ERROR(_impl->cast_to_type(target, &promoted));
1277+
*needs_jsonb = cast_introduced_null(*_impl->column, *promoted);
1278+
return Status::OK();
12441279
}
12451280

12461281
const PathInData& VariantPathBuilder::path() const {

‎be/src/storage/segment/variant/v2/variant_path_builder.h‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
#include <cstddef>
2121
#include <cstdint>
22+
#include <functional>
2223
#include <memory>
2324
#include <span>
2425

@@ -54,6 +55,10 @@ bool variant_path_type_contains_nothing(const DataTypePtr& type);
5455
// returns.
5556
class VariantPathBuilder final {
5657
public:
58+
// Resolve inferred widening against values held outside this builder, before either side
59+
// commits its cast. The resolver may replace the proposed type with JSONB.
60+
using InferredPromotion = std::function<Status(DataTypePtr*)>;
61+
5762
explicit VariantPathBuilder(PathInData path, size_t prefix_rows = 0);
5863
~VariantPathBuilder();
5964

@@ -62,9 +67,13 @@ class VariantPathBuilder final {
6267
VariantPathBuilder(const VariantPathBuilder&) = delete;
6368
VariantPathBuilder& operator=(const VariantPathBuilder&) = delete;
6469

65-
Status append(VariantRef value, size_t row);
70+
Status append(VariantRef value, size_t row, const InferredPromotion& on_promotion = {});
6671
Status complete_rows(size_t rows);
67-
Status convert_to(const DataTypePtr& storage_type);
72+
// Inferred conversions preserve values through JSONB fallback; explicit storage coercion
73+
// retains the default cast-null filtering.
74+
Status convert_to(const DataTypePtr& storage_type, bool filter_cast_nulls = true);
75+
// Probe inferred widening without changing values, types or rowids in retained chunks.
76+
Status check_inferred_promotion(const DataTypePtr& target_type, bool* needs_jsonb) const;
6877

6978
const PathInData& path() const;
7079
const DataTypePtr& type() const;

‎be/src/storage/segment/variant/variant_column_writer_impl.cpp‎

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -612,6 +612,11 @@ Status VariantSubcolumnWriter::_append_v2(const VariantColumnData& column, size_
612612
_chunk_rows = 0;
613613
}
614614

615+
VariantPathBuilder::InferredPromotion on_promotion;
616+
// Until a chunk is sealed, the active builder already holds every value to be checked.
617+
if (!_chunks.empty()) {
618+
on_promotion = [this](DataTypePtr* type) { return _promote_chunks(type); };
619+
}
615620
const auto append_encoded = [&](const ColumnVariantV2::ReadView& view, size_t begin) -> Status {
616621
DORIS_CHECK(!view.is_typed());
617622
for (size_t offset = 0; offset < num_rows; ++offset) {
@@ -621,7 +626,8 @@ Status VariantSubcolumnWriter::_append_v2(const VariantColumnData& column, size_
621626
const size_t input_row = begin + offset;
622627
const VariantRef value = view.value_at(input_row);
623628
if (!value.is_null()) {
624-
RETURN_IF_ERROR(_v2_builder->append(value, _num_rows + offset - _chunk_row_begin));
629+
RETURN_IF_ERROR(_v2_builder->append(value, _num_rows + offset - _chunk_row_begin,
630+
on_promotion));
625631
}
626632
}
627633
return Status::OK();
@@ -657,6 +663,30 @@ Status VariantSubcolumnWriter::_append_v2(const VariantColumnData& column, size_
657663
return Status::OK();
658664
}
659665

666+
Status VariantSubcolumnWriter::_promote_chunks(DataTypePtr* inferred_type) {
667+
// Decide against every retained value before changing any chunk. A small scale-1 Decimal may
668+
// fit scale 38 while a later large Decimal does not; committing the first cast would change
669+
// its JSONB representation before discovering that the whole path needs fallback.
670+
for (const auto& chunk : _chunks) {
671+
if (chunk.builder->non_null_rows() == 0) {
672+
continue;
673+
}
674+
bool needs_jsonb = false;
675+
RETURN_IF_ERROR(chunk.builder->check_inferred_promotion(*inferred_type, &needs_jsonb));
676+
if (needs_jsonb) {
677+
*inferred_type = std::make_shared<DataTypeJsonb>();
678+
break;
679+
}
680+
}
681+
for (auto& chunk : _chunks) {
682+
if (chunk.builder->non_null_rows() != 0) {
683+
RETURN_IF_ERROR(
684+
chunk.builder->convert_to(*inferred_type, false /* filter_cast_nulls */));
685+
}
686+
}
687+
return Status::OK();
688+
}
689+
660690
Status VariantSubcolumnWriter::_seal_chunk() {
661691
if (_v2_builder == nullptr) {
662692
return Status::OK();

0 commit comments

Comments
 (0)