Skip to content

Commit 66c9704

Browse files
committed
[fix](nereids) HBO: address doris-skills code review round 1
### What problem does this PR solve? Problem Summary: 按 doris-skills(doris-repo-review 方法论:风险扫描 → 分模块 并发 v4pro full-review → 共享 ledger 收敛)对 master..hbo-structinfo 全量 diff(25 文件)复审。合并后 A1 Major / A2 Minor / C1 Minor 及 Nit C2-C4、 N-B1 全部落实: - A1(Major,pinned filter-on-scan 谓词盲):注入 key 为 scan-group fingerprint(表 token 不含 filter 常量),pinned 绕过 learned 侧 ScanPlanStatistics 谓词匹配 → 同表同版本同裁剪下不同常量共享同一 pin。 该语义为 scan-group key 模型的固有权衡(SET 无 plan 上下文,谓词敏感 key 不可表达),修复=精确文档化(HboStatsCalculator 注释 + 设计文档 §4.1)+ 回归锁定:同扫描不同常量(R.b=1 vs b=2)共享 pin 且同样翻转, 防止未来误改 key 模型。 - A2/N-B2(Minor):HboUtils.genPlanStatisticsMap struct 模式下快照整体为 空(全部节点无 fingerprint)时单独 warn(原逻辑仅部分缺失时告警,最坏 诊断场景反而静默)。 - C1(Minor):putPinnedPlanStatistics 的 DB persist 移入 pinnedLoadLock, 与 DELETE 的 DB 删除同锁 —— 同键并发 SET/DELETE 的 DB 写序与内存序 一致,库行恒等于最后内存写入者(修复 round2 "last memory writer == last DB writer" 不成立的残余)。 - C2(Nit):pinned cache 上限语义对齐兄弟 hbo cache(num<=0 为无界而非 静默禁用注入),Config javadoc 注明。 - C3(Nit):HboStatisticsCommand fingerprint 统一小写(ROOT),SET 与 DELETE 均做 64-hex 校验 —— 大写/畸形 DELETE 不再静默无效。 - C4(Nit):builder 行数 Long.parseLong 溢出 → 清晰 ParseException。 - N-B1(Nit):EXPLAIN 注解块仅在物理层级输出,排除 PARSED/ANALYZED/ REWRITTEN 的误导性 "(no hbo fingerprint attached ...)" 回退。 - B-open(filter 发布 key 一致性)经代码证据排除:扫描父 filter 折叠进干净 scan 时不注册 nereids→planId 映射(仅建 SelectNode 路径注册),collect 不会覆盖 scan 条目;SelectNode filter 条目以其自身组 fp 发布,读侧无 任何 lookup 使用该 key,惰性无害。 验证:build.sh --fe EXIT=0(checkstyle 0 违规);hbo_structinfo_inject_test (含新增 b=2 共享 pin 断言)与 hbo_persist_internal_db_test 均 Test 1 suites, failed 0;手工负例:大写 SET→小写 DELETE 命中、畸形 DELETE 报错、 rows 溢出报 "out of range"、PARSED/REWRITTEN explain 无注解块而默认 explain 有块。 ### Release note None(实验特性内部修订,默认关闭) ### Check List (For Author) - Test: FE 构建通过;回归两套通过;手工负例验证 - Behavior changed: No - Does this need documentation: No
1 parent d95d7fb commit 66c9704

8 files changed

Lines changed: 61 additions & 13 deletions

File tree

‎fe/fe-common/src/main/java/org/apache/doris/common/Config.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2322,11 +2322,13 @@ public class Config extends ConfigBase {
23222322
/**
23232323
* Maximum number of manually injected (pinned) hbo statistics entries kept per FE. When the
23242324
* limit is exceeded the least recently used pinned entry is evicted (LRU); pinned entries are
2325-
* otherwise not expired automatically.
2325+
* otherwise not expired automatically. A non-positive value disables the LRU bound
2326+
* (unbounded), matching the other hbo caches. Takes effect at FE start (not hot-mutable).
23262327
*/
23272328
@ConfField(description = "The default setting is 5000. Maximum number of manually injected "
23282329
+ "(pinned) hbo statistics entries kept per FE; exceeding the limit evicts the least "
2329-
+ "recently used entry. Takes effect at FE start (not hot-mutable).")
2330+
+ "recently used entry; a non-positive value disables the bound (unbounded). Takes "
2331+
+ "effect at FE start (not hot-mutable).")
23302332
public static int hbo_pinned_stats_cache_num = 5000;
23312333

23322334
/**

‎fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1290,7 +1290,14 @@ public String getExplainString(ExplainOptions explainOptions) {
12901290
if (statementContext != null && !statementContext.getHints().isEmpty()) {
12911291
hint = getHintExplainString(statementContext.getHints());
12921292
}
1293-
if (ConnectContext.get() != null && ConnectContext.get().getSessionVariable().isShowHboFingerprint()
1293+
// the annotation block is meaningful only once a physical plan exists (fragment-form
1294+
// explain); parsed/analyzed/rewritten tree output must not carry the misleading
1295+
// "(no hbo fingerprint attached ...)" fallback
1296+
if (explainLevel != ExplainLevel.PARSED_PLAN
1297+
&& explainLevel != ExplainLevel.ANALYZED_PLAN
1298+
&& explainLevel != ExplainLevel.REWRITTEN_PLAN
1299+
&& ConnectContext.get() != null
1300+
&& ConnectContext.get().getSessionVariable().isShowHboFingerprint()
12941301
&& physicalPlan != null && cascadesContext != null) {
12951302
plan += appendHboFingerprintAnnotations();
12961303
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7114,7 +7114,12 @@ public LogicalPlan visitAdminSetFrontendConfig(DorisParser.AdminSetFrontendConfi
71147114
public LogicalPlan visitHboSetStatistics(DorisParser.HboSetStatisticsContext ctx) {
71157115
checkHboStatementWords(ctx.hbo, ctx.statistics);
71167116
String fingerprint = stripQuotes(ctx.key.getText());
7117-
long rows = Long.parseLong(ctx.rows.getText());
7117+
long rows;
7118+
try {
7119+
rows = Long.parseLong(ctx.rows.getText());
7120+
} catch (NumberFormatException e) {
7121+
throw new ParseException("hbo statistics rows out of range: " + ctx.rows.getText());
7122+
}
71187123
String structCanonical = "";
71197124
if (ctx.structWord != null && ctx.structCanonical != null) {
71207125
if (!"struct".equalsIgnoreCase(ctx.structWord.getText())) {

‎fe/fe-core/src/main/java/org/apache/doris/nereids/stats/HboPlanStatisticsManager.java‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -64,9 +64,13 @@ public class HboPlanStatisticsManager {
6464

6565
private HboPlanStatisticsProvider hboPlanStatisticsProvider;
6666
private HboPlanInfoProvider hboPlanInfoProvider;
67-
private final Cache<String, PinnedHboStatistics> pinnedPlanStatistics = Caffeine.newBuilder()
68-
.maximumSize(Math.max(Config.hbo_pinned_stats_cache_num, 0))
69-
.build();
67+
// LRU bound follows the sibling hbo caches (see HboPlanInfoProvider): a non-positive
68+
// hbo_pinned_stats_cache_num disables the bound instead of silently disabling injection
69+
private final Cache<String, PinnedHboStatistics> pinnedPlanStatistics =
70+
(Config.hbo_pinned_stats_cache_num > 0
71+
? Caffeine.newBuilder().maximumSize(Config.hbo_pinned_stats_cache_num)
72+
: Caffeine.newBuilder())
73+
.build();
7074
// whether pinned statistics have been loaded from the internal table (only when persistence
7175
// is enabled); planner/command threads access it concurrently
7276
private volatile boolean hboPinnedLoaded = false;
@@ -124,9 +128,12 @@ public void putPinnedPlanStatistics(String fingerprint, long rows, String nodeTy
124128
// a SET after a failed DELETE re-creates the entry: drop the deletion intent so a
125129
// pending load does not skip the re-created row
126130
pendingLoadTombstones.remove(fingerprint);
127-
}
128-
if (persistenceEnabled()) {
129-
HboStatisticsStore.persist(fingerprint, rows, nodeType, structCanonical, createTimeMs);
131+
// persist under the same lock as DELETE's DB removal, so concurrent same-key
132+
// SET/DELETE serialize their DB writes in the same order as their memory updates
133+
// and the stored row always matches the last memory writer
134+
if (persistenceEnabled()) {
135+
HboStatisticsStore.persist(fingerprint, rows, nodeType, structCanonical, createTimeMs);
136+
}
130137
}
131138
}
132139

‎fe/fe-core/src/main/java/org/apache/doris/nereids/stats/HboStatsCalculator.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,13 @@ private Statistics getStatsFromHboPlanStats(AbstractPlan planNode, Statistics de
101101
Optional<String> hash = planNodeAndHashOpt.get().getHash();
102102
if (hash.isPresent()) {
103103
// manually injected (pinned) statistics are authoritative and bypass the learned
104-
// entry matching
104+
// entry matching (ScanPlanStatistics predicate comparison). The pin key for a
105+
// filter-on-scan is the scan-group fingerprint, whose token carries table/ordinal/
106+
// partition-pruning/version but NOT the filter constants, so a pin is scan-wide: it
107+
// also overrides the output rows of the same scan shape with different filter
108+
// predicates. Manual pins are expert overrides keyed by the shape fingerprint
109+
// (see GroupStructInfo scan tokens and the design doc 4.1); keep this bypass, but
110+
// users must scope pins by version/pruning rather than by predicate constants.
105111
Optional<HboPlanStatisticsManager.PinnedHboStatistics> pinned = Env.getCurrentEnv()
106112
.getHboPlanStatisticsManager().getPinnedPlanStatistics(hash.get());
107113
if (pinned.isPresent()) {

‎fe/fe-core/src/main/java/org/apache/doris/nereids/stats/HboUtils.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -535,6 +535,7 @@ public static Map<PlanNodeAndHash, PlanStatisticsWithInputInfo> genPlanStatistic
535535
Map<Integer, String> nodeFingerprints) {
536536
Map<PlanNodeAndHash, PlanStatisticsWithInputInfo> outputPlanStatisticsMap = new HashMap<>();
537537
int missingFingerprints = 0;
538+
int skippedWithoutSnapshot = 0;
538539
for (TPlanNodeRuntimeStatsItem nodeStats : curPlanNodeRuntimeStats) {
539540
int nodeId = nodeStats.node_id;
540541
PhysicalPlan planNode = idToPlanMap.get(nodeId);
@@ -554,6 +555,7 @@ public static Map<PlanNodeAndHash, PlanStatisticsWithInputInfo> genPlanStatistic
554555
hash = inputTableStatisticsInfo.getHash();
555556
} else {
556557
// struct-info mode without a planning-time snapshot for this node: skip it
558+
skippedWithoutSnapshot++;
557559
if (nodeFingerprints != null && !nodeFingerprints.isEmpty()) {
558560
missingFingerprints++;
559561
}
@@ -570,6 +572,12 @@ public static Map<PlanNodeAndHash, PlanStatisticsWithInputInfo> genPlanStatistic
570572
if (missingFingerprints > 0) {
571573
LOG.warn("hbo struct fingerprint snapshot missing for {} of {} plan nodes",
572574
missingFingerprints, curPlanNodeRuntimeStats.size());
575+
} else if (skippedWithoutSnapshot > 0) {
576+
// the whole per-query snapshot is empty (e.g. every group resolved INVALID): the
577+
// worst diagnostic case must not be silent, unlike a partially populated snapshot
578+
LOG.warn("hbo struct fingerprint snapshot is empty: skipped {} of {} plan nodes; "
579+
+ "hbo statistics for this query will not be published",
580+
skippedWithoutSnapshot, curPlanNodeRuntimeStats.size());
573581
}
574582
return outputPlanStatisticsMap;
575583
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/HboStatisticsCommand.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,9 @@ public HboStatisticsCommand(Op op, String fingerprint, long rows) {
7575
public HboStatisticsCommand(Op op, String fingerprint, long rows, String structCanonical) {
7676
super(PlanType.HBO_STATISTICS_COMMAND);
7777
this.op = op;
78-
this.fingerprint = fingerprint;
78+
// sha256 hex fingerprints are lowercase everywhere (group struct info, read-side lookup);
79+
// normalize user input so an uppercase fingerprint cannot silently miss its entry
80+
this.fingerprint = fingerprint == null ? null : fingerprint.toLowerCase(java.util.Locale.ROOT);
7981
this.rows = rows;
8082
this.structCanonical = structCanonical == null ? "" : structCanonical;
8183
}
@@ -87,9 +89,11 @@ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception {
8789
throw new AnalysisException("Access denied: HBO statistics management requires ADMIN privilege");
8890
}
8991
HboPlanStatisticsManager hboManager = Env.getCurrentEnv().getHboPlanStatisticsManager();
92+
// validate the fingerprint shape on both operations: a malformed DELETE would otherwise
93+
// silently leave the pinned entry active (it can never match a stored fingerprint)
94+
validateFingerprint(fingerprint);
9095
switch (op) {
9196
case SET:
92-
validateFingerprint(fingerprint);
9397
if (rows < 0) {
9498
throw new AnalysisException("hbo statistics rows must be non-negative: " + rows);
9599
}

‎regression-test/suites/query_p0/hbo/hbo_structinfo_inject_test.groovy‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,15 @@ suite("hbo_structinfo_inject_test", "nonConcurrent") {
8383
assertTrue(linesAfter[joinIdxAfter + 1].contains("hbo_si_r"), nodeAfter)
8484
assertTrue((nodeAfter =~ /PhysicalFilter\[\d+\].*hboUsed=true/).find(), nodeAfter)
8585

86+
// The pin key for filter-on-scan is the scan-group fingerprint, whose token excludes the
87+
// filter constants: a different constant (R.b = 2) over the same scan shape/version shares
88+
// the pin and is flipped too (documented scan-wide override semantics, see GroupStructInfo
89+
// scan tokens). Lock the behavior in so an accidental predicate-sensitive key change is
90+
// caught.
91+
def otherConstantQuery = "select * from hbo_si_t join hbo_si_r on hbo_si_t.a = hbo_si_r.a where hbo_si_r.b = 2"
92+
def otherConstantText = firstFragment(explainText(otherConstantQuery))
93+
assertTrue(otherConstantText.contains("TABLE: hbo_test.hbo_si_r(hbo_si_r)"), otherConstantText)
94+
8695
} finally {
8796
sql """ HBO DELETE STATISTICS '${fingerprint}'; """
8897
sql "set global enable_hbo_info_collection=${prevInfoCollection};"

0 commit comments

Comments
 (0)