Skip to content

Commit 714d816

Browse files
morningmanclaude
andcommitted
[improvement](be) Reuse memory-tracked scratch when validating Variant payloads
### What problem does this PR solve? Issue Number: None Related PR: #68959 Problem Summary: validate_variant_payload() checks that the field values of every Variant object exactly tile the object's value section. To do that, require_valid_object() collected the byte spans of the field values into a fresh std::vector for every object it checked, nested ones included, and freed it when the check returned. ColumnVariantV2::insert_encoded_rows() (JNI readers, inserted encoded values, Variant element extraction), the EncodedRowsAppender that #68959 uses to import unshredded Parquet Variant values, and the native Arrow Variant reader validate every row this way. A 4096-row batch of object rows therefore paid at least 4096 allocations, and the scratch (16 bytes per field) came from plain malloc, which Doris memory tracking does not see. VariantBatchBuilder, which that importer replaced, checked the same intervals in a reused, tracked DorisVector. Add VariantPayloadValidator, which keeps the spans in one DorisVector used as a stack: an object pushes the span of each field after checking the field, so a nested object pushes above its ancestors and pops back before it returns. The checks, their order and their errors are unchanged. The batch callers hold one validator for the batch, and the encoded-row appender holds one for its whole life; validate_variant_payload() stays as the one-shot form for a single value. The class is declared in its own header so that core/field.h, which includes variant_field.h, does not pull in core/custom_allocator.h. ### Release note None ### Check List (For Author) - Test: Unit Test - VariantFieldTest: the reusable validator accepts and rejects the same payloads as the one-shot call, including a corrupt nested object followed by a valid one; its scratch capacity stays the same over 4096 rows; the scratch is visible to the memory tracker - ColumnVariantV2Test.EncodedRowsAppenderReusesValidationScratchAcrossChunks - Behavior changed: No - Does this need documentation: No Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1 parent 6e59b6e commit 714d816

7 files changed

Lines changed: 259 additions & 22 deletions

File tree

‎be/src/core/column/variant_v2/column_variant_v2.cpp‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@
4545
#include "core/value/variant/variant_canonical.h"
4646
#include "core/value/variant/variant_field.h"
4747
#include "core/value/variant/variant_parquet_encoding.h"
48+
#include "core/value/variant/variant_payload_validator.h"
4849
#include "exec/sort/sort_block.h"
4950

5051
namespace doris {
@@ -876,12 +877,13 @@ void ColumnVariantV2::sanity_check() const {
876877
const StringRef metadata = metadatas.get_data_at(id);
877878
validate_variant_metadata({metadata.data, metadata.size});
878879
}
880+
VariantPayloadValidator payload_validator;
879881
for (size_t row = 0; row < values.size(); ++row) {
880882
const uint32_t id = metadata_ids[row];
881883
DORIS_CHECK_LT(id, metadatas.size()) << "ColumnVariantV2 metadata id is out of range";
882884
const StringRef metadata = metadatas.get_data_at(id);
883-
validate_variant_payload({.metadata = {.data = metadata.data, .size = metadata.size},
884-
.value = values.get_data_at(row)});
885+
payload_validator.validate({.metadata = {.data = metadata.data, .size = metadata.size},
886+
.value = values.get_data_at(row)});
885887
}
886888
}
887889
}
@@ -973,12 +975,13 @@ void ColumnVariantV2::insert_encoded_rows( // NOLINT(readability-function-size)
973975

974976
// Validate every row before changing a typed destination or appending to an encoded one.
975977
// This keeps input errors atomic without introducing a second temporary column.
978+
VariantPayloadValidator payload_validator;
976979
if (data.meta_ids.empty()) {
977980
const VariantMetadataRef metadata = metadata_at(0);
978981
for (size_t row = 0; row < rows; ++row) {
979982
const uint32_t begin = data.value_offsets[row];
980983
const uint32_t end = data.value_offsets[row + 1];
981-
validate_variant_payload(
984+
payload_validator.validate(
982985
{.metadata = metadata,
983986
.value = {data.value_bytes.data + begin, static_cast<size_t>(end - begin)}});
984987
}
@@ -988,7 +991,7 @@ void ColumnVariantV2::insert_encoded_rows( // NOLINT(readability-function-size)
988991
DORIS_CHECK_LT(source_id, metadata_count) << "encoded row metadata id is out of range";
989992
const uint32_t begin = data.value_offsets[row];
990993
const uint32_t end = data.value_offsets[row + 1];
991-
validate_variant_payload(
994+
payload_validator.validate(
992995
{.metadata = metadata_at(source_id),
993996
.value = {data.value_bytes.data + begin, static_cast<size_t>(end - begin)}});
994997
}
@@ -1007,6 +1010,8 @@ struct ColumnVariantV2::EncodedRowsAppender::Impl {
10071010
ColumnVariantV2* column;
10081011
MetadataHashHeads metadata_hash_heads;
10091012
DorisVector<uint32_t> metadata_hash_next;
1013+
// Kept across append() calls so that every chunk a reader imports reuses one scratch.
1014+
VariantPayloadValidator payload_validator;
10101015
size_t expected_rows = 0;
10111016
bool initialized = false;
10121017
#ifdef BE_TEST
@@ -1025,6 +1030,10 @@ ColumnVariantV2::EncodedRowsAppender::EncodedRowsAppender(EncodedRowsAppender&&)
10251030
size_t ColumnVariantV2::EncodedRowsAppender::metadata_comparisons_for_test() const noexcept {
10261031
return _impl == nullptr ? 0 : _impl->metadata_comparisons;
10271032
}
1033+
1034+
size_t ColumnVariantV2::EncodedRowsAppender::validation_scratch_capacity_for_test() const noexcept {
1035+
return _impl == nullptr ? 0 : _impl->payload_validator.object_span_capacity_for_test();
1036+
}
10281037
#endif
10291038

10301039
void ColumnVariantV2::EncodedRowsAppender::append( // NOLINT(readability-function-size)
@@ -1094,7 +1103,7 @@ void ColumnVariantV2::EncodedRowsAppender::append( // NOLINT(readability-functio
10941103
if (!source_metadata_ids.empty()) {
10951104
source_metadata_ids[row] = source_metadata_id;
10961105
}
1097-
validate_variant_payload(value);
1106+
_impl->payload_validator.validate(value);
10981107
source_values[row] = value.value;
10991108
if (value.value.size > std::numeric_limits<size_t>::max() - total_value_bytes) {
11001109
throw Exception(ErrorCode::INVALID_ARGUMENT,

‎be/src/core/column/variant_v2/column_variant_v2.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,7 @@ class ColumnVariantV2 final : public COWHelper<IColumn, ColumnVariantV2> {
137137

138138
#ifdef BE_TEST
139139
size_t metadata_comparisons_for_test() const noexcept;
140+
size_t validation_scratch_capacity_for_test() const noexcept;
140141
#endif
141142

142143
private:

‎be/src/core/data_type_serde/data_type_variant_v2_serde.cpp‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@
5050
#include "core/value/jsonb_value.h"
5151
#include "core/value/variant/variant_batch_builder.h"
5252
#include "core/value/variant/variant_field.h"
53+
#include "core/value/variant/variant_payload_validator.h"
5354
#include "exprs/function/parse/variant_jsonb_parse.h"
5455
#include "exprs/function/parse/variant_string_parse.h"
5556
#include "util/jsonb_document.h"
@@ -541,6 +542,7 @@ Status DataTypeVariantV2SerDe::read_column_from_arrow(IColumn& column, const arr
541542
// A driver supplies untrusted bytes. Validate into a temporary batch before mutating
542543
// the destination, and copy buffers so the column survives Arrow batch release.
543544
VariantBatchBuilder decoded;
545+
VariantPayloadValidator payload_validator;
544546
for (int64_t i = 0; i < slice->length(); ++i) {
545547
auto row = decoded.begin_row();
546548
if (storage.IsNull(i)) {
@@ -556,7 +558,7 @@ Status DataTypeVariantV2SerDe::read_column_from_arrow(IColumn& column, const arr
556558
(VariantRef {.metadata = {.data = meta.data(), .size = meta.size()},
557559
.value = {value.data(), value.size()}});
558560
validate_variant_metadata(ref.metadata);
559-
validate_variant_payload(ref);
561+
payload_validator.validate(ref);
560562
row.add_value(ref);
561563
}
562564
row.finish();

‎be/src/core/value/variant/variant_field.cpp‎

Lines changed: 24 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,12 @@
2020
#include <algorithm>
2121
#include <cstring>
2222
#include <limits>
23+
#include <span>
2324
#include <utility>
24-
#include <vector>
2525

2626
#include "common/exception.h"
2727
#include "core/value/variant/variant_parquet_encoding.h"
28+
#include "core/value/variant/variant_payload_validator.h"
2829
#include "core/value/variant/variant_scalar.h"
2930
#include "util/utf8_check.h"
3031

@@ -128,13 +129,6 @@ void require_valid_primitive(VariantRef value) {
128129
throw Exception(ErrorCode::CORRUPTION, "VariantField contains an unknown primitive type");
129130
}
130131

131-
void require_exact_value(VariantRef value, uint32_t depth);
132-
133-
struct ObjectValueSpan {
134-
size_t offset;
135-
size_t size;
136-
};
137-
138132
size_t object_values_offset(VariantRef value, uint32_t count) {
139133
const uint8_t value_header =
140134
static_cast<uint8_t>(value.value.data[0]) >> VARIANT_VALUE_HEADER_SHIFT;
@@ -148,13 +142,15 @@ size_t object_values_offset(VariantRef value, uint32_t count) {
148142
(static_cast<size_t>(count) + 1) * offset_width;
149143
}
150144

151-
void require_valid_object(VariantRef value, uint32_t depth) {
145+
} // namespace
146+
147+
void VariantPayloadValidator::require_valid_object(VariantRef value, uint32_t depth) {
152148
const uint32_t count = value.num_elements();
153149
const size_t values_offset = object_values_offset(value, count);
154150
const char* values_begin = value.value.data + values_offset;
155151
const size_t values_size = value.value.size - values_offset;
156-
std::vector<ObjectValueSpan> spans;
157-
spans.reserve(count);
152+
// Entries below base belong to the ancestors of this object; see _object_spans.
153+
const size_t base = _object_spans.size();
158154

159155
StringRef previous_key;
160156
for (uint32_t index = 0; index < count; ++index) {
@@ -166,11 +162,12 @@ void require_valid_object(VariantRef value, uint32_t depth) {
166162
"VariantField object keys are not strictly ordered at field {}", index);
167163
}
168164
require_exact_value(child, depth + 1);
169-
spans.push_back({.offset = static_cast<size_t>(child.value.data - values_begin),
170-
.size = child.value.size});
165+
_object_spans.push_back({.offset = static_cast<size_t>(child.value.data - values_begin),
166+
.size = child.value.size});
171167
previous_key = key;
172168
}
173169

170+
const std::span<ObjectValueSpan> spans(_object_spans.data() + base, count);
174171
std::ranges::sort(spans, [](const ObjectValueSpan& left, const ObjectValueSpan& right) {
175172
return left.offset < right.offset;
176173
});
@@ -190,16 +187,17 @@ void require_valid_object(VariantRef value, uint32_t depth) {
190187
throw Exception(ErrorCode::CORRUPTION,
191188
"VariantField object values contain unreferenced trailing bytes");
192189
}
190+
_object_spans.resize(base);
193191
}
194192

195-
void require_valid_array(VariantRef value, uint32_t depth) {
193+
void VariantPayloadValidator::require_valid_array(VariantRef value, uint32_t depth) {
196194
const uint32_t count = value.num_elements();
197195
for (uint32_t index = 0; index < count; ++index) {
198196
require_exact_value(value.array_at(index), depth + 1);
199197
}
200198
}
201199

202-
void require_exact_value(VariantRef value, uint32_t depth) {
200+
void VariantPayloadValidator::require_exact_value(VariantRef value, uint32_t depth) {
203201
if (depth > VARIANT_MAX_NESTING_DEPTH) {
204202
throw Exception(ErrorCode::CORRUPTION,
205203
"VariantField value exceeds maximum nesting depth {}",
@@ -229,6 +227,8 @@ void require_exact_value(VariantRef value, uint32_t depth) {
229227
throw Exception(ErrorCode::CORRUPTION, "VariantField contains an unknown value type");
230228
}
231229

230+
namespace {
231+
232232
struct RowSlices {
233233
VariantMetadataRef metadata;
234234
VariantRef value;
@@ -276,9 +276,17 @@ void validate_variant_metadata(VariantMetadataRef metadata) {
276276
}
277277
}
278278

279-
void validate_variant_payload(VariantRef value) {
279+
void VariantPayloadValidator::validate(VariantRef value) {
280280
require_non_null(value.value, "value");
281+
// A call that threw in the middle of an object left the spans of its unfinished objects
282+
// behind, and the next call must not count them as its own.
283+
_object_spans.clear();
281284
require_exact_value(value, 0);
285+
DCHECK(_object_spans.empty());
286+
}
287+
288+
void validate_variant_payload(VariantRef value) {
289+
VariantPayloadValidator().validate(value);
282290
}
283291

284292
VariantField::VariantField(std::unique_ptr<char[]> data, size_t size) noexcept
Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
#pragma once
19+
20+
#include <cstddef>
21+
#include <cstdint>
22+
23+
#include "core/custom_allocator.h"
24+
#include "core/value/variant/variant_value.h"
25+
26+
namespace doris {
27+
28+
// Reusable form of validate_variant_payload() (variant_field.h), which is the one-shot call for a
29+
// single value. Checking an object needs the byte spans of its field values, and this validator
30+
// keeps them in one memory-tracked buffer that every later call reuses. A caller that validates
31+
// many rows holds one validator for all of them, so a batch of 4096 object rows grows the buffer a
32+
// few times instead of allocating once per object. The class is defined in variant_field.cpp next
33+
// to the checks it shares with validate_variant_payload(); it is declared apart from
34+
// variant_field.h so that core/field.h, which includes that header, does not also pull in
35+
// core/custom_allocator.h.
36+
class VariantPayloadValidator {
37+
public:
38+
// Validate exactly one recursive Variant payload, with the same checks and errors as
39+
// validate_variant_payload(). The referenced metadata must already have passed
40+
// validate_variant_metadata().
41+
void validate(VariantRef value);
42+
43+
#ifdef BE_TEST
44+
size_t object_span_capacity_for_test() const noexcept { return _object_spans.capacity(); }
45+
#endif
46+
47+
private:
48+
// The bytes one field value takes inside its object's value section.
49+
struct ObjectValueSpan {
50+
size_t offset;
51+
size_t size;
52+
};
53+
54+
void require_exact_value(VariantRef value, uint32_t depth);
55+
void require_valid_object(VariantRef value, uint32_t depth);
56+
void require_valid_array(VariantRef value, uint32_t depth);
57+
58+
// A stack shared by the objects on the path from the root to the value being checked. Each
59+
// object pushes the span of a field after checking that field, so a nested object pushes its
60+
// own spans above those of its ancestors and pops them before it returns. For
61+
// {"a":1,"b":{"c":2,"d":3},"e":4}, the stack holds [a] when the root starts checking "b";
62+
// "b" grows it to [a, c, d], checks its own two entries and pops back to [a], and the root
63+
// then pushes "b" and "e" and checks [a, b, e].
64+
DorisVector<ObjectValueSpan> _object_spans;
65+
};
66+
67+
} // namespace doris

‎be/test/core/column/column_variant_v2_test.cpp‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1226,6 +1226,34 @@ TEST(ColumnVariantV2Test, EncodedRowsAppenderInternsHighCardinalityMetadataAcros
12261226
column->sanity_check();
12271227
}
12281228

1229+
TEST(ColumnVariantV2Test, EncodedRowsAppenderReusesValidationScratchAcrossChunks) {
1230+
constexpr size_t FIELDS = 64;
1231+
constexpr size_t ROWS = 4096;
1232+
std::string json = R"({"nested":{"x":1,"y":[{"z":2}]})";
1233+
for (size_t field = 0; field < FIELDS; ++field) {
1234+
json += ",\"k" + std::to_string(field) + "\":" + std::to_string(field);
1235+
}
1236+
json.push_back('}');
1237+
const VariantField wide = encode_json(json);
1238+
const std::vector<VariantRef> rows(ROWS, wide.ref());
1239+
1240+
auto column = ColumnVariantV2::create();
1241+
auto appender = column->create_encoded_rows_appender();
1242+
EXPECT_EQ(appender.validation_scratch_capacity_for_test(), 0);
1243+
appender.append(rows);
1244+
const size_t capacity = appender.validation_scratch_capacity_for_test();
1245+
EXPECT_GE(capacity, FIELDS + 1);
1246+
appender.append(rows);
1247+
EXPECT_EQ(appender.validation_scratch_capacity_for_test(), capacity);
1248+
1249+
ASSERT_EQ(column->size(), 2 * ROWS);
1250+
JsonWriter expected;
1251+
to_json(wide.ref(), expected);
1252+
EXPECT_EQ(json_at(*column, 0), expected.value);
1253+
EXPECT_EQ(json_at(*column, 2 * ROWS - 1), expected.value);
1254+
column->sanity_check();
1255+
}
1256+
12291257
TEST(ColumnVariantV2Test, InvalidEncodedRowsDoNotPartiallyAppendEncodedState) {
12301258
const OwnedEncodedData invalid = late_invalid_encoded_rows();
12311259
for (const bool omit_single_metadata_ids : {false, true}) {

0 commit comments

Comments
 (0)