@@ -250,6 +250,9 @@ int64_t VariantColumnReader::get_metadata_size() const {
250250 }
251251 }
252252
253+ for (const auto & [path, meta] : _embedded_subcolumn_metas) {
254+ size += path.size () + meta.SpaceUsedLong ();
255+ }
253256 for (const auto & reader : *_subcolumns_meta_info) {
254257 size += reader->path .get_path ().size ();
255258 size += sizeof (SubcolumnMeta);
@@ -476,11 +479,11 @@ Status VariantColumnReader::_build_read_plan_flat_leaves(
476479 return Status::OK ();
477480 }
478481 // Case 2: bucketized sparse column path: __DORIS_VARIANT_SPARSE__.b{i}
479- if (rel.rfind (std::string (SPARSE_COLUMN_PATH ) + " .b" , 0 ) == 0 ) {
482+ // A suffix such as .b0tail belongs to a user column, not bucket zero.
483+ if (auto bucket = variant_util::variant_binary_column_bucket (rel, SPARSE_COLUMN_PATH );
484+ rel != SPARSE_COLUMN_PATH && bucket) {
480485 CHECK (_binary_column_reader->get_type () == BinaryColumnType::MULTIPLE_SPARSE );
481- // parse bucket index
482- uint32_t bucket_index = static_cast <uint32_t >(
483- atoi (rel.substr (std::string (SPARSE_COLUMN_PATH ).size () + 2 ).c_str ()));
486+ auto bucket_index = static_cast <uint32_t >(*bucket);
484487 const auto & reader = _binary_column_reader->select_reader (bucket_index);
485488 if (!reader) {
486489 return Status::NotFound (" bucket sparse column reader not found: {}" , rel);
@@ -496,10 +499,10 @@ Status VariantColumnReader::_build_read_plan_flat_leaves(
496499 }
497500
498501 // case 3: doc snapshot column
499- if (rel.find (DOC_VALUE_COLUMN_PATH ) != std::string::npos) {
502+ // Marker substrings must use the ordinary leaf/sparse read plan.
503+ if (auto bucket = variant_util::variant_binary_column_bucket (rel, DOC_VALUE_COLUMN_PATH )) {
500504 CHECK (_binary_column_reader->get_type () == BinaryColumnType::MULTIPLE_DOC_VALUE );
501- size_t bucket = rel.rfind (' b' );
502- uint32_t bucket_value = static_cast <uint32_t >(std::stoul (rel.substr (bucket + 1 )));
505+ auto bucket_value = static_cast <uint32_t >(*bucket);
503506 plan->kind = ReadKind::DOC_COMPACT ;
504507 plan->type = DataTypeFactory::instance ().create_data_type (target_col);
505508 plan->binary_column_reader = _binary_column_reader->select_reader (bucket_value);
@@ -1253,10 +1256,9 @@ Status VariantColumnReader::init(const ColumnReaderOptions& opts, ColumnMetaAcce
12531256 }
12541257
12551258 // case 2: bucketized sparse column
1256- std::string bucket_prefix = std::string (SPARSE_COLUMN_PATH ) + " .b" ;
1257- if (rel_str.starts_with (bucket_prefix)) {
1258- uint32_t idx =
1259- static_cast <uint32_t >(atoi (rel_str.substr (bucket_prefix.size ()).c_str ()));
1259+ // Only exact writer-generated bucket names are physical Map columns.
1260+ if (auto bucket = variant_util::variant_binary_column_bucket (rel_str, SPARSE_COLUMN_PATH )) {
1261+ auto idx = static_cast <uint32_t >(*bucket);
12601262 DCHECK (col.has_variant_statistics ()) << col.DebugString ();
12611263 if (should_record_path_stats) {
12621264 // Additively merge per-bucket sparse stats into the unified statistics.
@@ -1270,9 +1272,10 @@ Status VariantColumnReader::init(const ColumnReaderOptions& opts, ColumnMetaAcce
12701272 }
12711273
12721274 // case 3: doc snapshot column
1273- if (rel_str.find (DOC_VALUE_COLUMN_PATH ) != std::string::npos) {
1274- size_t bucket = rel_str.rfind (' b' );
1275- uint32_t bucket_value = static_cast <uint32_t >(std::stoi (rel_str.substr (bucket + 1 )));
1275+ // A user key such as x__DORIS_VARIANT_DOC_VALUE__y is an ordinary subcolumn.
1276+ if (auto bucket =
1277+ variant_util::variant_binary_column_bucket (rel_str, DOC_VALUE_COLUMN_PATH )) {
1278+ auto bucket_value = static_cast <uint32_t >(*bucket);
12761279 std::shared_ptr<ColumnReader> column_reader;
12771280 RETURN_IF_ERROR (ColumnReader::create (opts, col, num_rows, file_reader, &column_reader));
12781281 tmp_doc_value_readers[bucket_value] = std::move (column_reader);
@@ -1286,12 +1289,56 @@ Status VariantColumnReader::init(const ColumnReaderOptions& opts, ColumnMetaAcce
12861289 return Status::OK ();
12871290 };
12881291
1292+ auto add_subcolumn_meta = [&](const ColumnMetaPB& column_pb, int32_t ordinal) {
1293+ PathInData path;
1294+ path.from_protobuf (column_pb.column_path_info ());
1295+ // init subcolumns
1296+ auto relative_path = path.copy_pop_front ();
1297+ if (relative_path.empty ()) {
1298+ return ;
1299+ }
1300+ // Skip NestedGroup subcolumns (columns with ___DOR_ng___. prefix in path).
1301+ // NestedGroup columns only contain rows that have the nested array, not all rows.
1302+ // They need special handling via NestedGroupWholeIterator, not regular subcolumns.
1303+ const auto & leaf_path = relative_path.get_path ();
1304+ if (contains_nested_group_marker (leaf_path)) {
1305+ VLOG_DEBUG << " Skipping NestedGroup subcolumn: " << leaf_path;
1306+ return ;
1307+ }
1308+ // check the root is already a leaf node
1309+ if (should_record_path_stats && column_pb.has_none_null_size ()) {
1310+ _statistics->subcolumns_non_null_size .emplace (relative_path.get_path (),
1311+ column_pb.none_null_size ());
1312+ }
1313+ // 3.1.2 may store a flat JSON key like {"a.b": 1} as a single PathInData part.
1314+ // New compaction schema and query path expect a dot-split multi-part shape.
1315+ // Rebuild via the string constructor when the path has neither typed
1316+ // nor nested metadata, so the tree matches the new shape.
1317+ if (!relative_path.get_is_typed () && !relative_path.has_nested_part ()) {
1318+ relative_path = PathInData (relative_path.get_path ());
1319+ }
1320+ _subcolumns_meta_info->add (
1321+ relative_path,
1322+ SubcolumnMeta {
1323+ .file_column_type = DataTypeFactory::instance ().create_data_type (column_pb),
1324+ .footer_ordinal = ordinal});
1325+ };
1326+
12891327 // First try initialize sparse from root's embedded children_columns (new segments)
12901328 if (self_column_pb.children_columns_size () > 0 ) {
12911329 for (int i = 0 ; i < self_column_pb.children_columns_size (); ++i) {
12921330 const ColumnMetaPB& child_pb = self_column_pb.children_columns (i);
12931331 bool handled = false ;
12941332 RETURN_IF_ERROR (handle_sparse_meta (child_pb, &handled));
1333+ // Old V3 writers embedded user columns with marker substrings and omitted their
1334+ // path-index entries. Keep their metadata locally so query AND compaction can
1335+ // read the existing data pages through the normal subcolumn reader cache.
1336+ if (!handled && child_pb.has_column_path_info ()) {
1337+ PathInData path;
1338+ path.from_protobuf (child_pb.column_path_info ());
1339+ _embedded_subcolumn_metas.emplace (std::string (path.get_relative_path ()), child_pb);
1340+ add_subcolumn_meta (child_pb, -1 );
1341+ }
12951342 }
12961343 }
12971344
@@ -1316,12 +1363,7 @@ Status VariantColumnReader::init(const ColumnReaderOptions& opts, ColumnMetaAcce
13161363 continue ;
13171364 }
13181365 DCHECK (column_pb.has_column_path_info ());
1319- PathInData path;
1320- path.from_protobuf (column_pb.column_path_info ());
1321-
13221366 // init sparse column readers
1323- auto relative_sparse = path.copy_pop_front ();
1324- auto rel_str = relative_sparse.get_path ();
13251367 {
13261368 bool handled = false ;
13271369 RETURN_IF_ERROR (handle_sparse_meta (column_pb, &handled));
@@ -1330,36 +1372,7 @@ Status VariantColumnReader::init(const ColumnReaderOptions& opts, ColumnMetaAcce
13301372 }
13311373 }
13321374
1333- // init subcolumns
1334- auto relative_path = path.copy_pop_front ();
1335- if (relative_path.empty ()) {
1336- continue ;
1337- }
1338- // Skip NestedGroup subcolumns (columns with ___DOR_ng___. prefix in path).
1339- // NestedGroup columns only contain rows that have the nested array, not all rows.
1340- // They need special handling via NestedGroupWholeIterator, not regular subcolumns.
1341- const auto & leaf_path = relative_path.get_path ();
1342- if (contains_nested_group_marker (leaf_path)) {
1343- VLOG_DEBUG << " Skipping NestedGroup subcolumn: " << leaf_path;
1344- continue ;
1345- }
1346- // check the root is already a leaf node
1347- if (should_record_path_stats && column_pb.has_none_null_size ()) {
1348- _statistics->subcolumns_non_null_size .emplace (relative_path.get_path (),
1349- column_pb.none_null_size ());
1350- }
1351- // 3.1.2 may store a flat JSON key like {"a.b": 1} as a single PathInData part.
1352- // New compaction schema and query path expect a dot-split multi-part shape.
1353- // Rebuild via the string constructor when the path has neither typed
1354- // nor nested metadata, so the tree matches the new shape.
1355- if (!relative_path.get_is_typed () && !relative_path.has_nested_part ()) {
1356- relative_path = PathInData (relative_path.get_path ());
1357- }
1358- _subcolumns_meta_info->add (
1359- relative_path,
1360- SubcolumnMeta {
1361- .file_column_type = DataTypeFactory::instance ().create_data_type (column_pb),
1362- .footer_ordinal = ordinal});
1375+ add_subcolumn_meta (column_pb, ordinal);
13631376 }
13641377
13651378 // finalize bucket readers if any
@@ -1422,6 +1435,13 @@ Status VariantColumnReader::create_path_reader(
14221435 const io::FileReaderSPtr& file_reader, uint64_t num_rows,
14231436 std::shared_ptr<ColumnReader>* out, OlapReaderStatistics* stats,
14241437 const io::IOContext* source_io_ctx) {
1438+ // Old V3 marker-substring columns have neither a footer ordinal nor a path-index entry.
1439+ // This immutable metadata is populated once at segment initialization, with no per-row work.
1440+ if (auto it = _embedded_subcolumn_metas.find (relative_path.get_path ());
1441+ it != _embedded_subcolumn_metas.end ()) {
1442+ return ColumnReader::create (opts, it->second , num_rows, file_reader, out);
1443+ }
1444+
14251445 // 1) Try inline subcolumn meta if available (footer_ordinal >= 0)
14261446 const auto * node = get_subcolumn_meta_by_path (relative_path);
14271447 if (node != nullptr && node->data .footer_ordinal >= 0 ) {
0 commit comments