Skip to content

Commit 8096753

Browse files
seawindeDoris-Extras
authored andcommitted
[improvement](mtmv) Support union rewrite when the materialized view is not enough to provide all the data for the query (#33800)
When the materialized view is not enough to provide all the data for the query, if the materialized view is increment update by partition. we can union materialized view and origin query to reponse the query. this depends on #33362 such as materialized view def is as following: > CREATE MATERIALIZED VIEW mv_10086 > BUILD IMMEDIATE REFRESH AUTO ON MANUAL > partition by(l_shipdate) > DISTRIBUTED BY RANDOM BUCKETS 2 > PROPERTIES ('replication_num' = '1') > AS > select l_shipdate, o_orderdate, l_partkey, l_suppkey, sum(o_totalprice) as sum_total > from lineitem > left join orders on lineitem.l_orderkey = orders.o_orderkey and l_shipdate = o_orderdate > group by > l_shipdate, > o_orderdate, > l_partkey, > l_suppkey; the materialized view data is as following: +------------+-------------+-----------+-----------+-----------+ | l_shipdate | o_orderdate | l_partkey | l_suppkey | sum_total | +------------+-------------+-----------+-----------+-----------+ | 2023-10-18 | 2023-10-18 | 2 | 3 | 109.20 | | 2023-10-17 | 2023-10-17 | 2 | 3 | 99.50 | | 2023-10-19 | 2023-10-19 | 2 | 3 | 99.50 | +------------+-------------+-----------+-----------+-----------+ when we insert data to partition `2023-10-17`, if we run query as following ``` select l_shipdate, o_orderdate, l_partkey, l_suppkey, sum(o_totalprice) as sum_total from lineitem left join orders on lineitem.l_orderkey = orders.o_orderkey and l_shipdate = o_orderdate group by l_shipdate, o_orderdate, l_partkey, l_suppkey; ``` query rewrite by materialzied view will fail with message `Check partition query used validation fail` if we turn on the switch `SET enable_materialized_view_union_rewrite = true;` default true we run the query above again, it will success and will use union all materialized view and origin query to response the query correctly. the plan is as following: ``` | Explain String(Nereids Planner) | +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | PLAN FRAGMENT 0 | | OUTPUT EXPRS: | | l_shipdate[#52] | | o_orderdate[#53] | | l_partkey[#54] | | l_suppkey[#55] | | sum_total[#56] | | PARTITION: UNPARTITIONED | | | | HAS_COLO_PLAN_NODE: false | | | | VRESULT SINK | | MYSQL_PROTOCAL | | | | 11:VEXCHANGE | | offset: 0 | | distribute expr lists: | | | | PLAN FRAGMENT 1 | | | | PARTITION: HASH_PARTITIONED: l_shipdate[#42], o_orderdate[#43], l_partkey[#44], l_suppkey[#45] | | | | HAS_COLO_PLAN_NODE: false | | | | STREAM DATA SINK | | EXCHANGE ID: 11 | | UNPARTITIONED | | | | 10:VUNION(756) | | | | | |----9:VAGGREGATE (merge finalize)(753) | | | | output: sum(partial_sum(o_totalprice)[#46])[#51] | | | | group by: l_shipdate[#42], o_orderdate[#43], l_partkey[#44], l_suppkey[#45] | | | | cardinality=2 | | | | distribute expr lists: l_shipdate[#42], o_orderdate[#43], l_partkey[#44], l_suppkey[#45] | | | | | | | 8:VEXCHANGE | | | offset: 0 | | | distribute expr lists: l_shipdate[#42] | | | | | 1:VEXCHANGE | | offset: 0 | | distribute expr lists: | | | | PLAN FRAGMENT 2 | | | | PARTITION: HASH_PARTITIONED: o_orderkey[#21], o_orderdate[#25] | | | | HAS_COLO_PLAN_NODE: false | | | | STREAM DATA SINK | | EXCHANGE ID: 08 | | HASH_PARTITIONED: l_shipdate[#42], o_orderdate[#43], l_partkey[#44], l_suppkey[#45] | | | | 7:VAGGREGATE (update serialize)(747) | | | STREAMING | | | output: partial_sum(o_totalprice[#41])[#46] | | | group by: l_shipdate[#37], o_orderdate[#38], l_partkey[#39], l_suppkey[#40] | | | cardinality=2 | | | distribute expr lists: l_shipdate[#37] | | | | | 6:VHASH JOIN(741) | | | join op: RIGHT OUTER JOIN(PARTITIONED)[] | | | equal join conjunct: (o_orderkey[#21] = l_orderkey[#5]) | | | equal join conjunct: (o_orderdate[#25] = l_shipdate[#15]) | | | runtime filters: RF000[min_max] <- l_orderkey[#5](2/2/2048), RF001[bloom] <- l_orderkey[#5](2/2/2048), RF002[min_max] <- l_shipdate[#15](1/1/2048), RF003[bloom] <- l_shipdate[#15](1/1/2048) | | | cardinality=2 | | | vec output tuple id: 4 | | | output tuple id: 4 | | | vIntermediate tuple ids: 3 | | | hash output slot ids: 6 7 24 25 15 | | | final projections: l_shipdate[#36], o_orderdate[#32], l_partkey[#34], l_suppkey[#35], o_totalprice[#31] | | | final project output tuple id: 4 | | | distribute expr lists: o_orderkey[#21], o_orderdate[#25] | | | distribute expr lists: l_orderkey[#5], l_shipdate[#15] | | | | | |----3:VEXCHANGE | | | offset: 0 | | | distribute expr lists: l_orderkey[#5] | | | | | 5:VEXCHANGE | | offset: 0 | | distribute expr lists: | | | | PLAN FRAGMENT 3 | | | | PARTITION: RANDOM | | | | HAS_COLO_PLAN_NODE: false | | | | STREAM DATA SINK | | EXCHANGE ID: 05 | | HASH_PARTITIONED: o_orderkey[#21], o_orderdate[#25] | | | | 4:VOlapScanNode(722) | | TABLE: union_db.orders(orders), PREAGGREGATION: ON | | runtime filters: RF000[min_max] -> o_orderkey[#21], RF001[bloom] -> o_orderkey[#21], RF002[min_max] -> o_orderdate[#25], RF003[bloom] -> o_orderdate[#25] | | partitions=3/3 (p_20231017,p_20231018,p_20231019), tablets=9/9, tabletList=161188,161190,161192 ... | | cardinality=3, avgRowSize=0.0, numNodes=1 | | pushAggOp=NONE | | | | PLAN FRAGMENT 4 | | | | PARTITION: HASH_PARTITIONED: l_orderkey[#5] | | | | HAS_COLO_PLAN_NODE: false | | | | STREAM DATA SINK | | EXCHANGE ID: 03 | | HASH_PARTITIONED: l_orderkey[#5], l_shipdate[#15] | | | | 2:VOlapScanNode(729) | | TABLE: union_db.lineitem(lineitem), PREAGGREGATION: ON | | PREDICATES: (l_shipdate[#15] >= '2023-10-17') AND (l_shipdate[#15] < '2023-10-18') | | partitions=1/3 (p_20231017), tablets=3/3, tabletList=161223,161225,161227 | | cardinality=2, avgRowSize=0.0, numNodes=1 | | pushAggOp=NONE | | | | PLAN FRAGMENT 5 | | | | PARTITION: RANDOM | | | | HAS_COLO_PLAN_NODE: false | | | | STREAM DATA SINK | | EXCHANGE ID: 01 | | RANDOM | | | | 0:VOlapScanNode(718) | | TABLE: union_db.mv_10086(mv_10086), PREAGGREGATION: ON | | partitions=2/3 (p_20231018_20231019,p_20231019_20231020), tablets=4/4, tabletList=161251,161253,161265 ... | | cardinality=2, avgRowSize=0.0, numNodes=1 | | pushAggOp=NONE | | | | MaterializedView | | MaterializedViewRewriteSuccessAndChose: | | Names: mv_10086 | | MaterializedViewRewriteSuccessButNotChose: | | | | MaterializedViewRewriteFail: | +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ```
1 parent 00ff5f0 commit 8096753

16 files changed

Lines changed: 2282 additions & 112 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java‎

Lines changed: 22 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -17,27 +17,25 @@
1717

1818
package org.apache.doris.mtmv;
1919

20-
import org.apache.doris.catalog.Column;
2120
import org.apache.doris.catalog.MTMV;
22-
import org.apache.doris.catalog.Partition;
21+
import org.apache.doris.nereids.CascadesContext;
2322
import org.apache.doris.nereids.NereidsPlanner;
2423
import org.apache.doris.nereids.StatementContext;
25-
import org.apache.doris.nereids.analyzer.UnboundResultSink;
26-
import org.apache.doris.nereids.analyzer.UnboundTableSink;
24+
import org.apache.doris.nereids.jobs.executor.Rewriter;
2725
import org.apache.doris.nereids.parser.NereidsParser;
2826
import org.apache.doris.nereids.properties.PhysicalProperties;
27+
import org.apache.doris.nereids.rules.RuleType;
28+
import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils;
29+
import org.apache.doris.nereids.rules.rewrite.EliminateSort;
2930
import org.apache.doris.nereids.trees.plans.Plan;
3031
import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel;
3132
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
32-
import org.apache.doris.nereids.trees.plans.logical.LogicalTableSink;
33+
import org.apache.doris.nereids.trees.plans.logical.LogicalResultSink;
3334
import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter;
34-
import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor;
3535
import org.apache.doris.qe.ConnectContext;
3636
import org.apache.doris.qe.OriginStatement;
3737

38-
import com.google.common.collect.Lists;
39-
40-
import java.util.stream.Collectors;
38+
import com.google.common.collect.ImmutableList;
4139

4240
/**
4341
* The cache for materialized view cache
@@ -70,27 +68,25 @@ public static MTMVCache from(MTMV mtmv, ConnectContext connectContext) {
7068
if (mvSqlStatementContext.getConnectContext().getStatementContext() == null) {
7169
mvSqlStatementContext.getConnectContext().setStatementContext(mvSqlStatementContext);
7270
}
73-
unboundMvPlan = unboundMvPlan.accept(new DefaultPlanVisitor<LogicalPlan, Void>() {
74-
// convert to table sink to eliminate sort under table sink, because sort under result sink can not be
75-
// eliminated
76-
@Override
77-
public LogicalPlan visitUnboundResultSink(UnboundResultSink<? extends Plan> unboundResultSink,
78-
Void context) {
79-
return new UnboundTableSink<>(mtmv.getFullQualifiers(),
80-
mtmv.getBaseSchema().stream().map(Column::getName).collect(Collectors.toList()),
81-
Lists.newArrayList(),
82-
mtmv.getPartitions().stream().map(Partition::getName).collect(Collectors.toList()),
83-
unboundResultSink.child());
84-
}
85-
}, null);
71+
// Can not convert to table sink, because use the same column from different table when self join
72+
// the out slot is wrong
8673
Plan originPlan = planner.plan(unboundMvPlan, PhysicalProperties.ANY, ExplainLevel.REWRITTEN_PLAN);
87-
// eliminate logicalTableSink because sink operator is useless in query rewrite by materialized view
88-
Plan mvPlan = planner.getCascadesContext().getRewritePlan().accept(new DefaultPlanRewriter<Object>() {
74+
// Eliminate result sink because sink operator is useless in query rewrite by materialized view
75+
// and the top sort can also be removed
76+
Plan mvPlan = originPlan.accept(new DefaultPlanRewriter<Object>() {
8977
@Override
90-
public Plan visitLogicalTableSink(LogicalTableSink<? extends Plan> logicalTableSink, Object context) {
91-
return logicalTableSink.child().accept(this, context);
78+
public Plan visitLogicalResultSink(LogicalResultSink<? extends Plan> logicalResultSink, Object context) {
79+
return logicalResultSink.child().accept(this, context);
9280
}
9381
}, null);
82+
// Optimize by rules to remove top sort
83+
CascadesContext parentCascadesContext = CascadesContext.initContext(mvSqlStatementContext, mvPlan,
84+
PhysicalProperties.ANY);
85+
mvPlan = MaterializedViewUtils.rewriteByRules(parentCascadesContext, childContext -> {
86+
Rewriter.getCteChildrenRewriter(childContext,
87+
ImmutableList.of(Rewriter.custom(RuleType.ELIMINATE_SORT, EliminateSort::new))).execute();
88+
return childContext.getRewritePlan();
89+
}, mvPlan, originPlan);
9490
return new MTMVCache(mvPlan, originPlan);
9591
}
9692
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/CommonSubExpressionOpt.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -83,8 +83,9 @@ private List<List<NamedExpression>> computeMultiLayerProjections(
8383
// 'case slot whenClause2 END'
8484
// This is illegal.
8585
Expression rewritten = expr.accept(ExpressionReplacer.INSTANCE, aliasMap);
86-
Alias alias = new Alias(rewritten);
87-
aliasMap.put(expr, alias);
86+
// if rewritten is already alias, use it directly, because in materialized view rewriting
87+
// Should keep out slot immutably after rewritten successfully
88+
aliasMap.put(expr, rewritten instanceof Alias ? (Alias) rewritten : new Alias(rewritten));
8889
}
8990
});
9091
layer.addAll(aliasMap.values());

0 commit comments

Comments
 (0)