Skip to content

Commit d1ca460

Browse files
eldenmoonclaude
andcommitted
[refactor](variant) Keep escape flag on tablet schema and send storage paths from FE
### What problem does this PR solve? Issue Number: #69995 Related PR: #69995 Problem Summary: Address review comments on the escaped Variant key property. 1. The escape flag lives only on TabletSchema. Remove the TabletColumn copy, its propagation into derived subcolumns, and the TabletColumnObjectPool bypass. inherit_index now takes the owning schema's flag as a required argument. 2. FE translates Variant slot sub paths of escaped OLAP tables into storage spelling in PlanTranslatorContext.createSlotDesc. BE query code (olap_scanner, segment, predicate_collector, rowid_fetcher, vslot_ref) returns to master; make_variant_storage_path is removed. This also keeps query-cache normalization distinct for v['a.b'] and v['a']['b']. ### Release note None ### Check List (For Author) - Test: BE UT (TabletSchemaTest, schema_util, collection statistics, snii routing, path_in_data, VariantColumnWriterReaderTest), FE UT (VariantEscapeTableTest, QueryCacheNormalizerTest, DescriptorToThriftConverterTest, CreateTableCommandTest, MTMVPlanUtilTest, PropertyAnalyzerTest), variant_p0 and dotted_key_* with fuzzy session variables. - Behavior changed: No - Does this need documentation: No Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 3265322 commit d1ca460

27 files changed

Lines changed: 168 additions & 302 deletions

‎be/src/exec/common/variant_util.cpp‎

Lines changed: 18 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -100,12 +100,6 @@
100100

101101
namespace doris::variant_util {
102102

103-
void inherit_variant_path_encoding(const TabletSchema& schema, TabletColumn& column) {
104-
if (column.is_variant_type() || column.is_extracted_column()) {
105-
column.set_variant_escape_dots_in_keys(schema.variant_escape_dots_in_keys());
106-
}
107-
}
108-
109103
namespace {
110104

111105
PathInData make_full_subcolumn_path(const TabletColumnPtr& parent_column, std::string_view path) {
@@ -138,7 +132,6 @@ void append_empty_key_subcolumn_from_stats(VariantCompactionPaths& paths_set_inf
138132
subcolumn.set_aggregation_method(parent_column->aggregation());
139133
subcolumn.set_variant_max_subcolumns_count(parent_column->variant_max_subcolumns_count());
140134
subcolumn.set_variant_enable_doc_mode(parent_column->variant_enable_doc_mode());
141-
subcolumn.set_variant_escape_dots_in_keys(parent_column->variant_escape_dots_in_keys());
142135
subcolumn.set_is_nullable(true);
143136
output_schema->append_column(subcolumn);
144137
}
@@ -546,7 +539,6 @@ void inherit_column_attributes(const TabletColumn& source, TabletColumn& target,
546539
return;
547540
}
548541
target.set_aggregation_method(source.aggregation());
549-
target.set_variant_escape_dots_in_keys(source.variant_escape_dots_in_keys());
550542

551543
// 1. bloom filter
552544
if (is_bf_supported_by_fe_for_variant_subcolumn(target.type())) {
@@ -565,12 +557,14 @@ void inherit_column_attributes(const TabletColumn& source, TabletColumn& target,
565557
if (target.is_variant_type()) {
566558
for (auto& index : source_indexes) {
567559
auto index_info = std::make_shared<TabletIndex>(*index);
568-
index_info->set_escaped_escaped_index_suffix_path(target.path_info_ptr()->get_path(),
569-
source.variant_escape_dots_in_keys());
560+
index_info->set_escaped_escaped_index_suffix_path(
561+
target.path_info_ptr()->get_path(),
562+
(*target_schema)->variant_escape_dots_in_keys());
570563
indexes_to_add.emplace_back(std::move(index_info));
571564
}
572565
} else {
573-
inherit_index(source_indexes, indexes_to_add, target);
566+
inherit_index(source_indexes, indexes_to_add, target,
567+
(*target_schema)->variant_escape_dots_in_keys());
574568
}
575569
auto target_indexes = (*target_schema)
576570
->inverted_indexs(target.parent_unique_id(),
@@ -679,15 +673,13 @@ MutableColumnPtr create_variant_binary_column() {
679673
ColumnArray::ColumnOffsets::create());
680674
}
681675

682-
// Sparse/doc compaction read columns use the same path codec as their owning root.
683676
TabletColumn create_sparse_column(const TabletColumn& variant) {
684677
TabletColumn res;
685678
res.set_name(variant.name_lower_case() + "." + SPARSE_COLUMN_PATH);
686679
res.set_type(FieldType::OLAP_FIELD_TYPE_MAP);
687680
res.set_aggregation_method(variant.aggregation());
688681
res.set_path_info(PathInData {variant.name_lower_case() + "." + SPARSE_COLUMN_PATH});
689682
res.set_parent_unique_id(variant.unique_id());
690-
res.set_variant_escape_dots_in_keys(variant.variant_escape_dots_in_keys());
691683
res.set_default_value("{}");
692684
TabletColumn child_tcolumn;
693685
child_tcolumn.set_type(FieldType::OLAP_FIELD_TYPE_STRING);
@@ -704,7 +696,6 @@ TabletColumn create_sparse_shard_column(const TabletColumn& variant, int bucket_
704696
res.set_type(FieldType::OLAP_FIELD_TYPE_MAP);
705697
res.set_aggregation_method(variant.aggregation());
706698
res.set_parent_unique_id(variant.unique_id());
707-
res.set_variant_escape_dots_in_keys(variant.variant_escape_dots_in_keys());
708699
res.set_default_value("{}");
709700
PathInData path(name);
710701
res.set_path_info(path);
@@ -723,7 +714,6 @@ TabletColumn create_doc_value_column(const TabletColumn& variant, int bucket_ind
723714
res.set_type(FieldType::OLAP_FIELD_TYPE_MAP);
724715
res.set_aggregation_method(variant.aggregation());
725716
res.set_parent_unique_id(variant.unique_id());
726-
res.set_variant_escape_dots_in_keys(variant.variant_escape_dots_in_keys());
727717
res.set_default_value("{}");
728718
res.set_path_info(PathInData {name});
729719

@@ -742,7 +732,6 @@ TabletColumn create_doc_value_compaction_column(const TabletColumn& variant, int
742732
res.set_type(FieldType::OLAP_FIELD_TYPE_VARIANT);
743733
res.set_aggregation_method(variant.aggregation());
744734
res.set_parent_unique_id(variant.unique_id());
745-
res.set_variant_escape_dots_in_keys(variant.variant_escape_dots_in_keys());
746735
res.set_path_info(PathInData {name});
747736
res.set_is_nullable(true);
748737
res.set_variant_enable_doc_mode(true);
@@ -1058,7 +1047,8 @@ Status VariantCompactionUtil::get_compaction_nested_columns(
10581047
.path_info = full_path});
10591048
inherit_column_attributes(*parent_column, nested_column);
10601049
TabletIndexes sub_column_indexes;
1061-
inherit_index(parent_indexes, sub_column_indexes, nested_column);
1050+
inherit_index(parent_indexes, sub_column_indexes, nested_column,
1051+
output_schema->variant_escape_dots_in_keys());
10621052
paths_set_info.subcolumn_indexes.emplace(path.get_path(), std::move(sub_column_indexes));
10631053
output_schema->append_column(nested_column);
10641054
VLOG_DEBUG << "append nested column " << path.get_path();
@@ -1107,7 +1097,6 @@ void VariantCompactionUtil::get_compaction_subcolumns_from_subpaths(
11071097
subcolumn.set_variant_max_subcolumns_count(
11081098
parent_column->variant_max_subcolumns_count());
11091099
subcolumn.set_variant_enable_doc_mode(parent_column->variant_enable_doc_mode());
1110-
subcolumn.set_variant_escape_dots_in_keys(parent_column->variant_escape_dots_in_keys());
11111100
subcolumn.set_is_nullable(true);
11121101
output_schema->append_column(subcolumn);
11131102
VLOG_DEBUG << "append sub column " << subpath << " data type "
@@ -1124,7 +1113,8 @@ void VariantCompactionUtil::get_compaction_subcolumns_from_subpaths(
11241113
.path_info = column_path});
11251114
inherit_column_attributes(*parent_column, sub_column);
11261115
TabletIndexes sub_column_indexes;
1127-
inherit_index(parent_indexes, sub_column_indexes, sub_column);
1116+
inherit_index(parent_indexes, sub_column_indexes, sub_column,
1117+
target->variant_escape_dots_in_keys());
11281118
paths_set_info.subcolumn_indexes.emplace(subpath, std::move(sub_column_indexes));
11291119
output_schema->append_column(sub_column);
11301120
VLOG_DEBUG << "append sub column " << subpath << " data type " << data_type->get_name();
@@ -1154,7 +1144,8 @@ void VariantCompactionUtil::get_compaction_subcolumns_from_data_types(
11541144
.path_info = column_path});
11551145
inherit_column_attributes(*parent_column, sub_column);
11561146
TabletIndexes sub_column_indexes;
1157-
inherit_index(parent_indexes, sub_column_indexes, sub_column);
1147+
inherit_index(parent_indexes, sub_column_indexes, sub_column,
1148+
target->variant_escape_dots_in_keys());
11581149
paths_set_info.sub_path_set.emplace(path.get_path());
11591150
paths_set_info.subcolumn_indexes.emplace(path.get_path(), std::move(sub_column_indexes));
11601151
output_schema->append_column(sub_column);
@@ -1450,8 +1441,6 @@ bool generate_sub_column_info(const TabletSchema& schema, int32_t col_unique_id,
14501441
to_column->set_path_info(
14511442
PathInData(parent_column.name_lower_case() + "." + path, is_typed));
14521443
to_column->set_aggregation_method(parent_column.aggregation());
1453-
to_column->set_variant_escape_dots_in_keys(
1454-
parent_column.variant_escape_dots_in_keys());
14551444
to_column->set_is_nullable(true);
14561445
to_column->set_parent_unique_id(parent_column.unique_id());
14571446
if (from_column.is_decimal()) {
@@ -1474,14 +1463,15 @@ bool generate_sub_column_info(const TabletSchema& schema, int32_t col_unique_id,
14741463
auto index_ptr = std::make_shared<TabletIndex>(*index);
14751464
index_ptr->set_escaped_escaped_index_suffix_path(
14761465
sub_column_info->column.path_info_ptr()->get_path(),
1477-
parent_column.variant_escape_dots_in_keys());
1466+
schema.variant_escape_dots_in_keys());
14781467
sub_column_info->indexes.emplace_back(std::move(index_ptr));
14791468
}
14801469
}
14811470
// 2. find parent column's index
14821471
else if (const auto parent_index = schema.inverted_indexs(col_unique_id);
14831472
!parent_index.empty()) {
1484-
inherit_index(parent_index, sub_column_info->indexes, sub_column_info->column);
1473+
inherit_index(parent_index, sub_column_info->indexes, sub_column_info->column,
1474+
schema.variant_escape_dots_in_keys());
14851475
} else {
14861476
sub_column_info->indexes.clear();
14871477
}
@@ -1626,7 +1616,8 @@ bool inherit_index(const std::vector<const TabletIndex*>& parent_indexes,
16261616
}
16271617

16281618
bool inherit_index(const std::vector<const TabletIndex*>& parent_indexes,
1629-
TabletIndexes& subcolumns_indexes, const TabletColumn& column) {
1619+
TabletIndexes& subcolumns_indexes, const TabletColumn& column,
1620+
bool escape_dots) {
16301621
if (!column.is_extracted_column()) {
16311622
return false;
16321623
}
@@ -1645,12 +1636,10 @@ bool inherit_index(const std::vector<const TabletIndex*>& parent_indexes,
16451636
return false;
16461637
}
16471638
return inherit_index(parent_indexes, subcolumns_indexes, nested->type(),
1648-
column.path_info_ptr()->get_path(), true,
1649-
column.variant_escape_dots_in_keys());
1639+
column.path_info_ptr()->get_path(), true, escape_dots);
16501640
}
16511641
return inherit_index(parent_indexes, subcolumns_indexes, column.type(),
1652-
column.path_info_ptr()->get_path(), false,
1653-
column.variant_escape_dots_in_keys());
1642+
column.path_info_ptr()->get_path(), false, escape_dots);
16541643
}
16551644

16561645
bool inherit_index(const std::vector<const TabletIndex*>& parent_indexes,

‎be/src/exec/common/variant_util.h‎

Lines changed: 5 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -59,8 +59,6 @@ const std::string SPARSE_COLUMN_PATH = "__DORIS_VARIANT_SPARSE__";
5959
const std::string DOC_VALUE_COLUMN_PATH = "__DORIS_VARIANT_DOC_VALUE__";
6060
namespace doris::variant_util {
6161

62-
void inherit_variant_path_encoding(const TabletSchema& schema, TabletColumn& column);
63-
6462
using PathToNoneNullValues = std::unordered_map<std::string, int64_t>;
6563
using PathToDataTypes = std::unordered_map<PathInData, std::vector<DataTypePtr>, PathInData::Hash>;
6664

@@ -108,8 +106,6 @@ struct ExtraInfo {
108106
TabletColumn get_column_by_type(const DataTypePtr& data_type, const std::string& name,
109107
const ExtraInfo& ext_info);
110108

111-
// A retained Variant UID cannot change storage path encoding. New column identities are allowed.
112-
113109
// check if the tuple_paths has ambiguous paths
114110
// situation:
115111
// throw exception if there exists a prefix with matched names, but not matched structure (is Nested, number of dimensions).
@@ -163,18 +159,18 @@ uint32_t variant_binary_shard_of(const StringRef& path, uint32_t bucket_num);
163159

164160
void get_field_info(const Field& field, FieldInfo* info);
165161

166-
// inherit index from parent column
162+
// inherit index from parent column. escape_dots is the owning tablet schema's
163+
// variant_escape_dots_in_keys; it also escapes '%' in the index suffix.
167164
bool inherit_index(const std::vector<const TabletIndex*>& parent_indexes,
168165
TabletIndexes& sub_column_indexes, FieldType column_type,
169-
const std::string& suffix_path, bool is_array_nested_type = false,
170-
bool escape_dots = false);
166+
const std::string& suffix_path, bool is_array_nested_type, bool escape_dots);
171167

172168
bool inherit_index(const std::vector<const TabletIndex*>& parent_indexes,
173-
TabletIndexes& sub_column_indexes, const TabletColumn& column);
169+
TabletIndexes& sub_column_indexes, const TabletColumn& column, bool escape_dots);
174170

175171
bool inherit_index(const std::vector<const TabletIndex*>& parent_indexes,
176172
TabletIndexes& sub_column_indexes, const segment_v2::ColumnMetaPB& column_pb,
177-
bool escape_dots = false);
173+
bool escape_dots);
178174

179175
Status update_least_schema_internal(const std::map<PathInData, DataTypes>& subcolumns_types,
180176
TabletSchemaSPtr& common_schema, int32_t variant_col_unique_id,

‎be/src/exec/rowid_fetcher.cpp‎

Lines changed: 1 addition & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,6 @@
3030
#include <cstdint>
3131
#include <limits>
3232
#include <memory>
33-
#include <optional>
3433
#include <ostream>
3534
#include <string>
3635
#include <unordered_map>
@@ -44,7 +43,6 @@
4443
#include "core/column/column.h"
4544
#include "core/data_type/data_type_struct.h"
4645
#include "core/data_type_serde/data_type_serde.h"
47-
#include "exec/common/variant_util.h"
4846
#include "exec/scan/file_scanner_v2.h"
4947
#include "format_v2/table/lance_reader.h"
5048
#include "io/io_common.h"
@@ -1212,15 +1210,7 @@ Status RowIdStorageReader::read_doris_format_row(
12121210
"fetch request carries {} column descs for {} slots, slot {} has none",
12131211
fetch_columns.size(), slots.size(), slots[x].col_name());
12141212
}
1215-
std::optional<TabletColumn> escaped_column;
1216-
if (tablet->tablet_schema()->variant_escape_dots_in_keys()) {
1217-
// Standalone request columns carry no encoding. Bind escaped paths such as
1218-
// v['a.b'] on a local copy; legacy tablets can use the request column directly.
1219-
escaped_column.emplace(fetch_columns[x]);
1220-
variant_util::inherit_variant_path_encoding(*tablet->tablet_schema(),
1221-
*escaped_column);
1222-
}
1223-
const TabletColumn& read_column = escaped_column ? *escaped_column : fetch_columns[x];
1213+
const TabletColumn& read_column = fetch_columns[x];
12241214
set_slot_access_paths(slots[x], read_column, iterator_item.storage_read_options);
12251215
RETURN_IF_ERROR(segment->seek_and_read_by_rowid(read_column, &slots[x], row_ids, column,
12261216
iterator_item.storage_read_options,

‎be/src/exec/scan/olap_scanner.cpp‎

Lines changed: 10 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -433,6 +433,9 @@ Status OlapScanner::_init_tablet_reader_params(
433433
_tablet_reader_params.score_runtime = _score_runtime;
434434
_tablet_reader_params.output_columns = ((OlapScanLocalState*)_local_state)->_output_column_ids;
435435
_tablet_reader_params.ann_topn_runtime = _ann_topn_runtime;
436+
for (const auto& ele : ((OlapScanLocalState*)_local_state)->_cast_types_for_variants) {
437+
_tablet_reader_params.target_cast_type_for_variants[ele.first] = ele.second;
438+
};
436439
for (const auto& predicates : slot_to_predicates) {
437440
for (const auto& predicate : predicates.second) {
438441
_tablet_reader_params.predicates.push_back(predicate->clone(predicate->column_id()));
@@ -587,10 +590,8 @@ Status OlapScanner::_init_variant_columns() {
587590
}
588591
// Materialized paths are absent from the persisted frontend schema. Build their transient
589592
// read-schema entries from the slot type.
590-
const auto& parent = tablet_schema->column_by_uid(slot->col_unique_id());
591-
const PathInData path =
592-
make_variant_storage_path(parent.name_lower_case(), slot->column_paths(),
593-
parent.variant_escape_dots_in_keys());
593+
const PathInData path(tablet_schema->column_by_uid(slot->col_unique_id()).name_lower_case(),
594+
slot->column_paths());
594595
// Keep transient paths nullable so an absent path preserves the existing NULL result.
595596
TabletColumn subcol = variant_util::get_column_by_type(
596597
make_nullable(slot->type()), path.get_path(),
@@ -620,10 +621,9 @@ Status OlapScanner::_init_read_schema() {
620621
int32_t index = 0;
621622
auto& tablet_schema = _tablet_reader_params.tablet_schema;
622623
if (slot->type()->get_primitive_type() == PrimitiveType::TYPE_VARIANT) {
623-
const auto& parent = tablet_schema->column_by_uid(slot->col_unique_id());
624-
index = tablet_schema->field_index(
625-
make_variant_storage_path(parent.name_lower_case(), slot->column_paths(),
626-
parent.variant_escape_dots_in_keys()));
624+
index = tablet_schema->field_index(PathInData(
625+
tablet_schema->column_by_uid(slot->col_unique_id()).name_lower_case(),
626+
slot->column_paths()));
627627
} else {
628628
index = slot->col_unique_id() >= 0 ? tablet_schema->field_index(slot->col_unique_id())
629629
: tablet_schema->field_index(slot->col_name());
@@ -634,13 +634,9 @@ Status OlapScanner::_init_read_schema() {
634634
"field name is invalid. field={}, field_name_to_index={}, col_unique_id={}",
635635
slot->col_name(), tablet_schema->get_all_field_names(), slot->col_unique_id());
636636
}
637+
637638
const auto& column = tablet_schema->column(index);
638639
auto* olap_local_state = static_cast<OlapScanLocalState*>(_local_state);
639-
const auto& cast_types = olap_local_state->_cast_types_for_variants;
640-
if (auto it = cast_types.find(slot->col_name()); it != cast_types.end()) {
641-
// Predicates use storage column names after logical paths have been encoded.
642-
_tablet_reader_params.target_cast_type_for_variants[column.name()] = it->second;
643-
}
644640
const auto& olap_scan_node = olap_local_state->olap_scan_node();
645641
if (olap_scan_node.__isset.extra_key_column_slot_ids &&
646642
olap_scan_node.extra_key_column_slot_ids.contains(slot->id())) {
@@ -657,6 +653,7 @@ Status OlapScanner::_init_read_schema() {
657653
if (!slot->all_access_paths().empty()) {
658654
_tablet_reader_params.all_access_paths.insert({unique_id, slot->all_access_paths()});
659655
}
656+
660657
if (!slot->predicate_access_paths().empty()) {
661658
_tablet_reader_params.predicate_access_paths.insert(
662659
{unique_id, slot->predicate_access_paths()});

0 commit comments

Comments
 (0)