Skip to content

Commit 0f24500

Browse files
authored
[fix](Nereids) RewriteCteChildren not work with cost based rewritter (#26326)
we use a map to record rewrite cte children result to avoid rewrite twice in cost based rewritter. However, we record cte outer and inner in one map, and use null as outer result's key, use cte id as inner result's key. This is wrong, because every anchor has an outer, and we could only record one outer. So when we use the cache in cost based rewritter, we get wrong outer plan from the cache. Then the error will be thrown as below: ``` Caused by: java.lang.IllegalArgumentException: Stats for CTE: CTEId#1 not found at com.google.common.base.Preconditions.checkArgument(Preconditions.java:143) ~[guava-32.1.2-jre.jar:?] at org.apache.doris.nereids.stats.StatsCalculator.visitLogicalCTEConsumer(StatsCalculator.java:1049) ~[classes/:?] at org.apache.doris.nereids.stats.StatsCalculator.visitLogicalCTEConsumer(StatsCalculator.java:147) ~[classes/:?] at org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer.accept(LogicalCTEConsumer.java:111) ~[classes/:?] at org.apache.doris.nereids.stats.StatsCalculator.estimate(StatsCalculator.java:222) ~[classes/:?] at org.apache.doris.nereids.stats.StatsCalculator.estimate(StatsCalculator.java:200) ~[classes/:?] at org.apache.doris.nereids.jobs.cascades.DeriveStatsJob.execute(DeriveStatsJob.java:108) ~[classes/:?] at org.apache.doris.nereids.jobs.scheduler.SimpleJobScheduler.executeJobPool(SimpleJobScheduler.java:39) ~[classes/:?] at org.apache.doris.nereids.jobs.executor.Optimizer.execute(Optimizer.java:51) ~[classes/:?] at org.apache.doris.nereids.jobs.rewrite.CostBasedRewriteJob.getCost(CostBasedRewriteJob.java:98) ~[classes/:?] at org.apache.doris.nereids.jobs.rewrite.CostBasedRewriteJob.execute(CostBasedRewriteJob.java:64) ~[classes/:?] at org.apache.doris.nereids.jobs.executor.AbstractBatchJobExecutor.execute(AbstractBatchJobExecutor.java:119) ~[classes/:?] at org.apache.doris.nereids.rules.rewrite.RewriteCteChildren.visit(RewriteCteChildren.java:72) ~[classes/:?] at org.apache.doris.nereids.rules.rewrite.RewriteCteChildren.visit(RewriteCteChildren.java:56) ~[classes/:?] at org.apache.doris.nereids.trees.plans.visitor.PlanVisitor.visitLogicalSink(PlanVisitor.java:118) ~[classes/:?] at org.apache.doris.nereids.trees.plans.visitor.SinkVisitor.visitLogicalResultSink(SinkVisitor.java:72) ~[classes/:?] at org.apache.doris.nereids.trees.plans.logical.LogicalResultSink.accept(LogicalResultSink.java:58) ~[classes/:?] at org.apache.doris.nereids.rules.rewrite.RewriteCteChildren.visitLogicalCTEAnchor(RewriteCteChildren.java:86) ~[classes/:?] at org.apache.doris.nereids.rules.rewrite.RewriteCteChildren.visitLogicalCTEAnchor(RewriteCteChildren.java:56) ~[classes/:?] at org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor.accept(LogicalCTEAnchor.java:60) ~[classes/:?] at org.apache.doris.nereids.rules.rewrite.RewriteCteChildren.visitLogicalCTEAnchor(RewriteCteChildren.java:86) ~[classes/:?] at org.apache.doris.nereids.rules.rewrite.RewriteCteChildren.visitLogicalCTEAnchor(RewriteCteChildren.java:56) ~[classes/:?] at org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor.accept(LogicalCTEAnchor.java:60) ~[classes/:?] at org.apache.doris.nereids.rules.rewrite.RewriteCteChildren.rewriteRoot(RewriteCteChildren.java:67) ~[classes/:?] at org.apache.doris.nereids.jobs.rewrite.CustomRewriteJob.execute(CustomRewriteJob.java:58) ~[classes/:?] at org.apache.doris.nereids.jobs.executor.AbstractBatchJobExecutor.execute(AbstractBatchJobExecutor.java:119) ~[classes/:?] at org.apache.doris.nereids.NereidsPlanner.rewrite(NereidsPlanner.java:275) ~[classes/:?] at org.apache.doris.nereids.NereidsPlanner.plan(NereidsPlanner.java:218) ~[classes/:?] at org.apache.doris.nereids.NereidsPlanner.plan(NereidsPlanner.java:118) ~[classes/:?] at org.apache.doris.nereids.trees.plans.commands.ExplainCommand.run(ExplainCommand.java:81) ~[classes/:?] at org.apache.doris.qe.StmtExecutor.executeByNereids(StmtExecutor.java:550) ~[classes/:?] ```
1 parent 6892fc9 commit 0f24500

4 files changed

Lines changed: 20 additions & 10 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,8 @@ public class StatementContext {
8585
private final Map<CTEId, Set<RelationId>> cteIdToConsumerUnderProjects = new HashMap<>();
8686
// Used to update consumer's stats
8787
private final Map<CTEId, List<Pair<Map<Slot, Slot>, Group>>> cteIdToConsumerGroup = new HashMap<>();
88-
private final Map<CTEId, LogicalPlan> rewrittenCtePlan = new HashMap<>();
88+
private final Map<CTEId, LogicalPlan> rewrittenCteProducer = new HashMap<>();
89+
private final Map<CTEId, LogicalPlan> rewrittenCteConsumer = new HashMap<>();
8990
private final Map<String, Hint> hintMap = Maps.newLinkedHashMap();
9091
private final Set<String> viewDdlSqlSet = Sets.newHashSet();
9192

@@ -230,8 +231,12 @@ public Map<CTEId, List<Pair<Map<Slot, Slot>, Group>>> getCteIdToConsumerGroup()
230231
return cteIdToConsumerGroup;
231232
}
232233

233-
public Map<CTEId, LogicalPlan> getRewrittenCtePlan() {
234-
return rewrittenCtePlan;
234+
public Map<CTEId, LogicalPlan> getRewrittenCteProducer() {
235+
return rewrittenCteProducer;
236+
}
237+
238+
public Map<CTEId, LogicalPlan> getRewrittenCteConsumer() {
239+
return rewrittenCteConsumer;
235240
}
236241

237242
public void addViewDdlSql(String ddlSql) {

‎fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/CostBasedRewriteJob.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ private Optional<Pair<Cost, GroupExpression>> getCost(CascadesContext currentCtx
8989
CascadesContext rootCtx = currentCtx.getRoot();
9090
if (rootCtx.getRewritePlan() instanceof LogicalCTEAnchor) {
9191
// set subtree rewrite cache
92-
currentCtx.getStatementContext().getRewrittenCtePlan()
92+
currentCtx.getStatementContext().getRewrittenCteProducer()
9393
.put(currentCtx.getCurrentTree().orElse(null), (LogicalPlan) cboCtx.getRewritePlan());
9494
// Do Whole tree rewrite
9595
CascadesContext rootCtxCopy = CascadesContext.newCurrentTreeContext(rootCtx);

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/RewriteCteChildren.java‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -77,14 +77,14 @@ public Plan visit(Plan plan, CascadesContext context) {
7777
public Plan visitLogicalCTEAnchor(LogicalCTEAnchor<? extends Plan, ? extends Plan> cteAnchor,
7878
CascadesContext cascadesContext) {
7979
LogicalPlan outer;
80-
if (cascadesContext.getStatementContext().getRewrittenCtePlan().containsKey(null)) {
81-
outer = cascadesContext.getStatementContext().getRewrittenCtePlan().get(null);
80+
if (cascadesContext.getStatementContext().getRewrittenCteConsumer().containsKey(cteAnchor.getCteId())) {
81+
outer = cascadesContext.getStatementContext().getRewrittenCteProducer().get(cteAnchor.getCteId());
8282
} else {
8383
CascadesContext outerCascadesCtx = CascadesContext.newSubtreeContext(
8484
Optional.empty(), cascadesContext, cteAnchor.child(1),
8585
cascadesContext.getCurrentJobContext().getRequiredProperties());
8686
outer = (LogicalPlan) cteAnchor.child(1).accept(this, outerCascadesCtx);
87-
cascadesContext.getStatementContext().getRewrittenCtePlan().put(null, outer);
87+
cascadesContext.getStatementContext().getRewrittenCteConsumer().put(cteAnchor.getCteId(), outer);
8888
}
8989
boolean reserveAnchor = outer.anyMatch(p -> {
9090
if (p instanceof LogicalCTEConsumer) {
@@ -104,8 +104,8 @@ public Plan visitLogicalCTEAnchor(LogicalCTEAnchor<? extends Plan, ? extends Pla
104104
public Plan visitLogicalCTEProducer(LogicalCTEProducer<? extends Plan> cteProducer,
105105
CascadesContext cascadesContext) {
106106
LogicalPlan child;
107-
if (cascadesContext.getStatementContext().getRewrittenCtePlan().containsKey(cteProducer.getCteId())) {
108-
child = cascadesContext.getStatementContext().getRewrittenCtePlan().get(cteProducer.getCteId());
107+
if (cascadesContext.getStatementContext().getRewrittenCteProducer().containsKey(cteProducer.getCteId())) {
108+
child = cascadesContext.getStatementContext().getRewrittenCteProducer().get(cteProducer.getCteId());
109109
} else {
110110
child = (LogicalPlan) cteProducer.child();
111111
child = tryToConstructFilter(cascadesContext, cteProducer.getCteId(), child);
@@ -118,7 +118,7 @@ public Plan visitLogicalCTEProducer(LogicalCTEProducer<? extends Plan> cteProduc
118118
CascadesContext rewrittenCtx = CascadesContext.newSubtreeContext(
119119
Optional.of(cteProducer.getCteId()), cascadesContext, child, PhysicalProperties.ANY);
120120
child = (LogicalPlan) child.accept(this, rewrittenCtx);
121-
cascadesContext.getStatementContext().getRewrittenCtePlan().put(cteProducer.getCteId(), child);
121+
cascadesContext.getStatementContext().getRewrittenCteProducer().put(cteProducer.getCteId(), child);
122122
}
123123
return cteProducer.withChildren(child);
124124
}

‎regression-test/suites/nereids_syntax_p0/cte.groovy‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -324,5 +324,10 @@ suite("cte") {
324324
) tab
325325
WHERE Id IN (1, 2)
326326
"""
327+
328+
// rewrite cte children should work well with cost based rewrite rule. rely on rewrite rule: InferSetOperatorDistinct
329+
sql """
330+
WITH cte_0 AS ( SELECT 1 AS a ), cte_1 AS ( SELECT 1 AS a ) select * from cte_0, cte_1 union select * from cte_0, cte_1
331+
"""
327332
}
328333

0 commit comments

Comments
 (0)