Skip to content

Commit d5749a0

Browse files
eldenmoonclaude
andcommitted
[fix](variant) Identify Variant internal columns by exact path
### What problem does this PR solve? Issue Number: N/A Related PR: #69995 Problem Summary: Several BE paths identified the internal sparse and doc-value columns of a Variant by substring, so a user key such as x__DORIS_VARIANT_SPARSE__y was treated as an internal column. With storage_format V3 the key's metadata was embedded in the root without a path-index entry and the reader ignored it, so its values read as NULL and a later compaction dropped them. FULL compaction could also pass such a scalar column to the sparse statistics code, which expects a Map, and a doc-marker key made the reader throw from std::stoi. Recognize only the relative names writers generate (the single sparse column and canonical .b<n> sparse and doc buckets) with one helper, and use it at every site. Read ordinary children embedded by older V3 writers as regular subcolumns; this is per-segment metadata work with no per-row cost. A key exactly equal to an internal name is out of scope. ### Release note Fix wrong reads, compaction failures and possible data loss for Variant keys that contain the internal sparse or doc-value column names, and make subcolumns hidden by older V3 segments readable again. ### Check List (For Author) - Test: Unit Test and Regression test. - BE UT: exact identity against writer-generated names and malformed bucket suffixes, V3 metadata classification, statistics, inline round trip, and recovery of a V3 segment written with the old classifier. Affected test classes: 894 passed, 4 skipped. - Regression: reserved_internal_path_key, DUP/MOW x V2/V3 x ordinary/doc x 1/3 shards, load-time materialization and two FULL compactions, checking each row's value and the whole Variant. - variant_p0: all suites pass except two that fail the same way on unmodified master. - Behavior changed: Yes. Keys containing internal column names are ordinary paths. - Does this need documentation: No. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent a37fc36 commit d5749a0

14 files changed

Lines changed: 1105 additions & 75 deletions

‎be/src/exec/common/variant_util.cpp‎

Lines changed: 35 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333

3434
#include <algorithm>
3535
#include <cassert>
36+
#include <charconv>
3637
#include <cstddef>
3738
#include <cstdint>
3839
#include <cstring>
@@ -138,6 +139,34 @@ void append_empty_key_subcolumn_from_stats(VariantCompactionPaths& paths_set_inf
138139

139140
} // namespace
140141

142+
std::optional<int32_t> variant_binary_column_bucket(std::string_view relative_path,
143+
std::string_view internal_name) {
144+
if (relative_path == SPARSE_COLUMN_PATH && internal_name == SPARSE_COLUMN_PATH) {
145+
return 0;
146+
}
147+
// A user key such as x__DORIS_VARIANT_SPARSE__y or a nested marker is ordinary data.
148+
if (!relative_path.starts_with(internal_name)) {
149+
return std::nullopt;
150+
}
151+
auto suffix = relative_path.substr(internal_name.size());
152+
if (!suffix.starts_with(".b")) {
153+
return std::nullopt;
154+
}
155+
suffix.remove_prefix(2);
156+
// Match exactly the decimal names produced by std::to_string(bucket_index), including
157+
// the suffix boundary: .b01, .b-1, .b1tail and .b1.child are all user paths.
158+
if (suffix.empty() || suffix.front() < '0' || suffix.front() > '9' ||
159+
(suffix.size() > 1 && suffix.front() == '0')) {
160+
return std::nullopt;
161+
}
162+
int32_t bucket = 0;
163+
const auto [end, error] = std::from_chars(suffix.data(), suffix.data() + suffix.size(), bucket);
164+
if (error != std::errc() || end != suffix.data() + suffix.size()) {
165+
return std::nullopt;
166+
}
167+
return bucket;
168+
}
169+
141170
// NestedGroup's physical children and offsets are produced by NestedGroupWriteProvider, not by
142171
// appending TabletSchema extracted columns here. This predicate keeps only ordinary Variant paths
143172
// that are outside the NG tree, for example `v.owner` beside `v.items[*]`.
@@ -146,8 +175,8 @@ bool is_regular_path_outside_nested_group(const PathInData& path) {
146175
return !relative_path.empty() && !path.get_is_typed() && !path.has_nested_part() &&
147176
!segment_v2::contains_nested_group_marker(relative_path) &&
148177
!segment_v2::is_root_nested_group_path(relative_path) &&
149-
relative_path != SPARSE_COLUMN_PATH &&
150-
relative_path.find(DOC_VALUE_COLUMN_PATH) == std::string::npos;
178+
!variant_binary_column_bucket(relative_path, SPARSE_COLUMN_PATH) &&
179+
!variant_binary_column_bucket(relative_path, DOC_VALUE_COLUMN_PATH);
151180
}
152181

153182
bool should_materialize_nested_group_regular_subcolumns(
@@ -908,10 +937,10 @@ Status VariantCompactionUtil::check_path_stats(const std::vector<RowsetSharedPtr
908937
// check no extended schema in output rowset
909938
for (const auto& column : output->tablet_schema()->columns()) {
910939
if (column->is_extracted_column()) {
911-
const auto& name = column->name();
912-
if (name.find("." + DOC_VALUE_COLUMN_PATH + ".") != std::string::npos ||
913-
name.find("." + SPARSE_COLUMN_PATH + ".") != std::string::npos ||
914-
name.ends_with("." + SPARSE_COLUMN_PATH)) {
940+
const auto relative_path = column->path_info_ptr()->get_relative_path();
941+
// Only physical binary columns are allowed here; marker substrings are extracted data.
942+
if (variant_binary_column_bucket(relative_path, DOC_VALUE_COLUMN_PATH) ||
943+
variant_binary_column_bucket(relative_path, SPARSE_COLUMN_PATH)) {
915944
continue;
916945
}
917946
return Status::InternalError("Unexpected extracted column {} in output rowset",

‎be/src/exec/common/variant_util.h‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
#include <cstddef>
2424
#include <cstdint>
2525
#include <map>
26+
#include <optional>
2627
#include <string>
2728
#include <string_view>
2829
#include <unordered_map>
@@ -59,6 +60,12 @@ const std::string SPARSE_COLUMN_PATH = "__DORIS_VARIANT_SPARSE__";
5960
const std::string DOC_VALUE_COLUMN_PATH = "__DORIS_VARIANT_DOC_VALUE__";
6061
namespace doris::variant_util {
6162

63+
// Recognize a relative physical path, never a full path or a marker substring.
64+
// Writers emit the single sparse name or `<internal_name>.b<canonical nonnegative int>`.
65+
// Doc columns always have a bucket suffix. Return the bucket (0 for single sparse).
66+
std::optional<int32_t> variant_binary_column_bucket(std::string_view relative_path,
67+
std::string_view internal_name);
68+
6269
using PathToNoneNullValues = std::unordered_map<std::string, int64_t>;
6370
using PathToDataTypes = std::unordered_map<PathInData, std::vector<DataTypePtr>, PathInData::Hash>;
6471

‎be/src/storage/segment/column_writer.cpp‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -337,12 +337,14 @@ Status ColumnWriter::create_variant_writer(const ColumnWriterOptions& opts,
337337
const TabletColumn* column, io::FileWriter* file_writer,
338338
std::unique_ptr<ColumnWriter>* writer) {
339339
// Variant extracted columns have two kinds of physical writers:
340-
// - Doc-value snapshot column (`...__DORIS_VARIANT_DOC_VALUE__...`): use `VariantDocCompactWriter`
340+
// - Doc-value snapshot column (`<root>.__DORIS_VARIANT_DOC_VALUE__.b<i>`): use `VariantDocCompactWriter`
341341
// to store the doc snapshot in a compact binary form.
342342
// - Regular extracted subcolumns: use `VariantSubcolumnWriter`.
343343
// The root VARIANT column itself uses `VariantColumnWriter`.
344344
if (column->is_extracted_column()) {
345-
if (column->name().find(DOC_VALUE_COLUMN_PATH) != std::string::npos) {
345+
// User paths containing the marker must retain the ordinary subcolumn writer.
346+
if (variant_util::variant_binary_column_bucket(column->path_info_ptr()->get_relative_path(),
347+
DOC_VALUE_COLUMN_PATH)) {
346348
*writer = std::make_unique<VariantDocCompactWriter>(
347349
opts, std::make_shared<TabletColumn>(*column));
348350
return Status::OK();

‎be/src/storage/segment/variant/variant_column_reader.cpp‎

Lines changed: 69 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -250,6 +250,9 @@ int64_t VariantColumnReader::get_metadata_size() const {
250250
}
251251
}
252252

253+
for (const auto& [path, meta] : _embedded_subcolumn_metas) {
254+
size += path.size() + meta.SpaceUsedLong();
255+
}
253256
for (const auto& reader : *_subcolumns_meta_info) {
254257
size += reader->path.get_path().size();
255258
size += sizeof(SubcolumnMeta);
@@ -476,11 +479,11 @@ Status VariantColumnReader::_build_read_plan_flat_leaves(
476479
return Status::OK();
477480
}
478481
// Case 2: bucketized sparse column path: __DORIS_VARIANT_SPARSE__.b{i}
479-
if (rel.rfind(std::string(SPARSE_COLUMN_PATH) + ".b", 0) == 0) {
482+
// A suffix such as .b0tail belongs to a user column, not bucket zero.
483+
if (auto bucket = variant_util::variant_binary_column_bucket(rel, SPARSE_COLUMN_PATH);
484+
rel != SPARSE_COLUMN_PATH && bucket) {
480485
CHECK(_binary_column_reader->get_type() == BinaryColumnType::MULTIPLE_SPARSE);
481-
// parse bucket index
482-
uint32_t bucket_index = static_cast<uint32_t>(
483-
atoi(rel.substr(std::string(SPARSE_COLUMN_PATH).size() + 2).c_str()));
486+
auto bucket_index = static_cast<uint32_t>(*bucket);
484487
const auto& reader = _binary_column_reader->select_reader(bucket_index);
485488
if (!reader) {
486489
return Status::NotFound("bucket sparse column reader not found: {}", rel);
@@ -496,10 +499,10 @@ Status VariantColumnReader::_build_read_plan_flat_leaves(
496499
}
497500

498501
// case 3: doc snapshot column
499-
if (rel.find(DOC_VALUE_COLUMN_PATH) != std::string::npos) {
502+
// Marker substrings must use the ordinary leaf/sparse read plan.
503+
if (auto bucket = variant_util::variant_binary_column_bucket(rel, DOC_VALUE_COLUMN_PATH)) {
500504
CHECK(_binary_column_reader->get_type() == BinaryColumnType::MULTIPLE_DOC_VALUE);
501-
size_t bucket = rel.rfind('b');
502-
uint32_t bucket_value = static_cast<uint32_t>(std::stoul(rel.substr(bucket + 1)));
505+
auto bucket_value = static_cast<uint32_t>(*bucket);
503506
plan->kind = ReadKind::DOC_COMPACT;
504507
plan->type = DataTypeFactory::instance().create_data_type(target_col);
505508
plan->binary_column_reader = _binary_column_reader->select_reader(bucket_value);
@@ -1253,10 +1256,9 @@ Status VariantColumnReader::init(const ColumnReaderOptions& opts, ColumnMetaAcce
12531256
}
12541257

12551258
// case 2: bucketized sparse column
1256-
std::string bucket_prefix = std::string(SPARSE_COLUMN_PATH) + ".b";
1257-
if (rel_str.starts_with(bucket_prefix)) {
1258-
uint32_t idx =
1259-
static_cast<uint32_t>(atoi(rel_str.substr(bucket_prefix.size()).c_str()));
1259+
// Only exact writer-generated bucket names are physical Map columns.
1260+
if (auto bucket = variant_util::variant_binary_column_bucket(rel_str, SPARSE_COLUMN_PATH)) {
1261+
auto idx = static_cast<uint32_t>(*bucket);
12601262
DCHECK(col.has_variant_statistics()) << col.DebugString();
12611263
if (should_record_path_stats) {
12621264
// Additively merge per-bucket sparse stats into the unified statistics.
@@ -1270,9 +1272,10 @@ Status VariantColumnReader::init(const ColumnReaderOptions& opts, ColumnMetaAcce
12701272
}
12711273

12721274
// case 3: doc snapshot column
1273-
if (rel_str.find(DOC_VALUE_COLUMN_PATH) != std::string::npos) {
1274-
size_t bucket = rel_str.rfind('b');
1275-
uint32_t bucket_value = static_cast<uint32_t>(std::stoi(rel_str.substr(bucket + 1)));
1275+
// A user key such as x__DORIS_VARIANT_DOC_VALUE__y is an ordinary subcolumn.
1276+
if (auto bucket =
1277+
variant_util::variant_binary_column_bucket(rel_str, DOC_VALUE_COLUMN_PATH)) {
1278+
auto bucket_value = static_cast<uint32_t>(*bucket);
12761279
std::shared_ptr<ColumnReader> column_reader;
12771280
RETURN_IF_ERROR(ColumnReader::create(opts, col, num_rows, file_reader, &column_reader));
12781281
tmp_doc_value_readers[bucket_value] = std::move(column_reader);
@@ -1286,12 +1289,56 @@ Status VariantColumnReader::init(const ColumnReaderOptions& opts, ColumnMetaAcce
12861289
return Status::OK();
12871290
};
12881291

1292+
auto add_subcolumn_meta = [&](const ColumnMetaPB& column_pb, int32_t ordinal) {
1293+
PathInData path;
1294+
path.from_protobuf(column_pb.column_path_info());
1295+
// init subcolumns
1296+
auto relative_path = path.copy_pop_front();
1297+
if (relative_path.empty()) {
1298+
return;
1299+
}
1300+
// Skip NestedGroup subcolumns (columns with ___DOR_ng___. prefix in path).
1301+
// NestedGroup columns only contain rows that have the nested array, not all rows.
1302+
// They need special handling via NestedGroupWholeIterator, not regular subcolumns.
1303+
const auto& leaf_path = relative_path.get_path();
1304+
if (contains_nested_group_marker(leaf_path)) {
1305+
VLOG_DEBUG << "Skipping NestedGroup subcolumn: " << leaf_path;
1306+
return;
1307+
}
1308+
// check the root is already a leaf node
1309+
if (should_record_path_stats && column_pb.has_none_null_size()) {
1310+
_statistics->subcolumns_non_null_size.emplace(relative_path.get_path(),
1311+
column_pb.none_null_size());
1312+
}
1313+
// 3.1.2 may store a flat JSON key like {"a.b": 1} as a single PathInData part.
1314+
// New compaction schema and query path expect a dot-split multi-part shape.
1315+
// Rebuild via the string constructor when the path has neither typed
1316+
// nor nested metadata, so the tree matches the new shape.
1317+
if (!relative_path.get_is_typed() && !relative_path.has_nested_part()) {
1318+
relative_path = PathInData(relative_path.get_path());
1319+
}
1320+
_subcolumns_meta_info->add(
1321+
relative_path,
1322+
SubcolumnMeta {
1323+
.file_column_type = DataTypeFactory::instance().create_data_type(column_pb),
1324+
.footer_ordinal = ordinal});
1325+
};
1326+
12891327
// First try initialize sparse from root's embedded children_columns (new segments)
12901328
if (self_column_pb.children_columns_size() > 0) {
12911329
for (int i = 0; i < self_column_pb.children_columns_size(); ++i) {
12921330
const ColumnMetaPB& child_pb = self_column_pb.children_columns(i);
12931331
bool handled = false;
12941332
RETURN_IF_ERROR(handle_sparse_meta(child_pb, &handled));
1333+
// Old V3 writers embedded user columns with marker substrings and omitted their
1334+
// path-index entries. Keep their metadata locally so query AND compaction can
1335+
// read the existing data pages through the normal subcolumn reader cache.
1336+
if (!handled && child_pb.has_column_path_info()) {
1337+
PathInData path;
1338+
path.from_protobuf(child_pb.column_path_info());
1339+
_embedded_subcolumn_metas.emplace(std::string(path.get_relative_path()), child_pb);
1340+
add_subcolumn_meta(child_pb, -1);
1341+
}
12951342
}
12961343
}
12971344

@@ -1316,12 +1363,7 @@ Status VariantColumnReader::init(const ColumnReaderOptions& opts, ColumnMetaAcce
13161363
continue;
13171364
}
13181365
DCHECK(column_pb.has_column_path_info());
1319-
PathInData path;
1320-
path.from_protobuf(column_pb.column_path_info());
1321-
13221366
// init sparse column readers
1323-
auto relative_sparse = path.copy_pop_front();
1324-
auto rel_str = relative_sparse.get_path();
13251367
{
13261368
bool handled = false;
13271369
RETURN_IF_ERROR(handle_sparse_meta(column_pb, &handled));
@@ -1330,36 +1372,7 @@ Status VariantColumnReader::init(const ColumnReaderOptions& opts, ColumnMetaAcce
13301372
}
13311373
}
13321374

1333-
// init subcolumns
1334-
auto relative_path = path.copy_pop_front();
1335-
if (relative_path.empty()) {
1336-
continue;
1337-
}
1338-
// Skip NestedGroup subcolumns (columns with ___DOR_ng___. prefix in path).
1339-
// NestedGroup columns only contain rows that have the nested array, not all rows.
1340-
// They need special handling via NestedGroupWholeIterator, not regular subcolumns.
1341-
const auto& leaf_path = relative_path.get_path();
1342-
if (contains_nested_group_marker(leaf_path)) {
1343-
VLOG_DEBUG << "Skipping NestedGroup subcolumn: " << leaf_path;
1344-
continue;
1345-
}
1346-
// check the root is already a leaf node
1347-
if (should_record_path_stats && column_pb.has_none_null_size()) {
1348-
_statistics->subcolumns_non_null_size.emplace(relative_path.get_path(),
1349-
column_pb.none_null_size());
1350-
}
1351-
// 3.1.2 may store a flat JSON key like {"a.b": 1} as a single PathInData part.
1352-
// New compaction schema and query path expect a dot-split multi-part shape.
1353-
// Rebuild via the string constructor when the path has neither typed
1354-
// nor nested metadata, so the tree matches the new shape.
1355-
if (!relative_path.get_is_typed() && !relative_path.has_nested_part()) {
1356-
relative_path = PathInData(relative_path.get_path());
1357-
}
1358-
_subcolumns_meta_info->add(
1359-
relative_path,
1360-
SubcolumnMeta {
1361-
.file_column_type = DataTypeFactory::instance().create_data_type(column_pb),
1362-
.footer_ordinal = ordinal});
1375+
add_subcolumn_meta(column_pb, ordinal);
13631376
}
13641377

13651378
// finalize bucket readers if any
@@ -1422,6 +1435,13 @@ Status VariantColumnReader::create_path_reader(
14221435
const io::FileReaderSPtr& file_reader, uint64_t num_rows,
14231436
std::shared_ptr<ColumnReader>* out, OlapReaderStatistics* stats,
14241437
const io::IOContext* source_io_ctx) {
1438+
// Old V3 marker-substring columns have neither a footer ordinal nor a path-index entry.
1439+
// This immutable metadata is populated once at segment initialization, with no per-row work.
1440+
if (auto it = _embedded_subcolumn_metas.find(relative_path.get_path());
1441+
it != _embedded_subcolumn_metas.end()) {
1442+
return ColumnReader::create(opts, it->second, num_rows, file_reader, out);
1443+
}
1444+
14251445
// 1) Try inline subcolumn meta if available (footer_ordinal >= 0)
14261446
const auto* node = get_subcolumn_meta_by_path(relative_path);
14271447
if (node != nullptr && node->data.footer_ordinal >= 0) {

‎be/src/storage/segment/variant/variant_column_reader.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -421,6 +421,8 @@ class VariantColumnReader : public ColumnReader {
421421
// Protect `_subcolumns_meta_info` and `_statistics` when loading external meta.
422422
mutable std::shared_mutex _subcolumns_meta_mutex;
423423
std::unique_ptr<SubcolumnColumnMetaInfo> _subcolumns_meta_info;
424+
// Ordinary user columns embedded by the old V3 substring classifier. Immutable after init.
425+
std::unordered_map<std::string, ColumnMetaPB> _embedded_subcolumn_metas;
424426
// Sparse column readers (single or bucketized)
425427
std::shared_ptr<BinaryColumnReader> _binary_column_reader;
426428
std::shared_ptr<ColumnReader> _root_column_reader;

‎be/src/storage/segment/variant/variant_column_writer_impl.cpp‎

Lines changed: 8 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@
1919
#include <gen_cpp/segment_v2.pb.h>
2020

2121
#include <algorithm>
22-
#include <charconv>
2322
#include <memory>
2423
#include <ranges>
2524
#include <span>
@@ -71,19 +70,15 @@
7170
namespace doris::segment_v2 {
7271

7372
namespace {
74-
Status parse_doc_compact_bucket(std::string_view path, int* bucket) {
73+
Status parse_doc_compact_bucket(const PathInData& path, int* bucket) {
7574
DORIS_CHECK(bucket != nullptr);
76-
const std::string marker = "." + DOC_VALUE_COLUMN_PATH + ".b";
77-
const size_t marker_pos = path.rfind(marker);
78-
if (marker_pos == std::string_view::npos) {
79-
return Status::Corruption("Invalid Variant doc compact path {}", path);
80-
}
81-
const std::string_view suffix = path.substr(marker_pos + marker.size());
82-
const auto [end, error] =
83-
std::from_chars(suffix.data(), suffix.data() + suffix.size(), *bucket);
84-
if (error != std::errc() || end != suffix.data() + suffix.size() || *bucket < 0) {
85-
return Status::Corruption("Invalid Variant doc compact bucket in path {}", path);
75+
const auto index = variant_util::variant_binary_column_bucket(path.get_relative_path(),
76+
DOC_VALUE_COLUMN_PATH);
77+
// Reuse writer dispatch's exact identity check; nested markers are ordinary user paths.
78+
if (!index) {
79+
return Status::Corruption("Invalid Variant doc compact path {}", path.get_path());
8680
}
81+
*bucket = *index;
8782
return Status::OK();
8883
}
8984

@@ -1023,8 +1018,7 @@ Status VariantDocCompactWriter::_finalize_v2(const TabletColumn& parent_column,
10231018
_v2_shredder.reset();
10241019

10251020
int bucket_value = -1;
1026-
RETURN_IF_ERROR(
1027-
parse_doc_compact_bucket(get_column()->path_info_ptr()->get_path(), &bucket_value));
1021+
RETURN_IF_ERROR(parse_doc_compact_bucket(*get_column()->path_info_ptr(), &bucket_value));
10281022
if (cast_set<size_t>(bucket_value) >= shredded.binary_buckets.size()) {
10291023
return Status::Corruption("Invalid Variant doc compact bucket {} in path {}", bucket_value,
10301024
get_column()->path_info_ptr()->get_path());

‎be/src/storage/segment/variant/variant_ext_meta_writer.cpp‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -105,9 +105,12 @@ static void classify_columns(const SegmentFooterPB& footer, ColumnClassification
105105
std::string rel_path = rel.get_path();
106106
int32_t root_uid = col.column_path_info().parrent_column_unique_id();
107107

108-
if (rel_path.find("__DORIS_VARIANT_SPARSE__") != std::string::npos) {
108+
// Substring matches embed user columns (e.g. x__DORIS_VARIANT_SPARSE__y)
109+
// without an external index entry, making their committed values invisible.
110+
if (variant_util::variant_binary_column_bucket(rel_path, SPARSE_COLUMN_PATH)) {
109111
classification.pending_sparse[root_uid].emplace_back(col);
110-
} else if (rel_path.find("__DORIS_VARIANT_DOC_VALUE__") != std::string::npos) {
112+
} else if (variant_util::variant_binary_column_bucket(rel_path,
113+
DOC_VALUE_COLUMN_PATH)) {
111114
classification.pending_doc_value[root_uid].emplace_back(col);
112115
} else {
113116
classification.pending_subcols[root_uid].emplace_back(rel_path, col);

0 commit comments

Comments
 (0)