Skip to content

Commit 0d07e3d

Browse files
authored
[fix](mtmv) Fix written by mv successfully but not get mv lock. If use mv, data maybe wrong (#40173)
When refresh partition mv, meanwhile, the query that can be written successfully by the same mv, if we run the query, the result maybe wrong. this pr fix this.
1 parent a2f2854 commit 0d07e3d

5 files changed

Lines changed: 181 additions & 3 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,11 @@ public MTMV() {
113113
mvRwLock = new ReentrantReadWriteLock(true);
114114
}
115115

116+
@Override
117+
public boolean needReadLockWhenPlan() {
118+
return true;
119+
}
120+
116121
public MTMVRefreshInfo getRefreshInfo() {
117122
readMvLock();
118123
try {

‎fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -441,7 +441,8 @@ public synchronized void addTableReadLock(TableIf tableIf) {
441441
String fullTableName = tableIf.getNameWithFullQualifiers();
442442
String resourceName = "tableReadLock(" + fullTableName + ")";
443443
plannerResources.push(new CloseableResource(
444-
resourceName, Thread.currentThread().getName(), originStatement.originStmt, tableIf::readUnlock));
444+
resourceName, Thread.currentThread().getName(),
445+
originStatement == null ? null : originStatement.originStmt, tableIf::readUnlock));
445446
}
446447

447448
/** releasePlannerResources */

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -262,6 +262,12 @@ protected List<Plan> doRewrite(StructInfo queryStructInfo, CascadesContext casca
262262
// Rewrite query by view
263263
rewrittenPlan = rewriteQueryByView(matchMode, queryStructInfo, viewStructInfo, viewToQuerySlotMapping,
264264
rewrittenPlan, materializationContext, cascadesContext);
265+
// If rewrite successfully, try to get mv read lock to avoid data inconsistent,
266+
// try to get lock which should added before RBO
267+
if (materializationContext instanceof AsyncMaterializationContext && !materializationContext.isSuccess()) {
268+
cascadesContext.getStatementContext()
269+
.addTableReadLock(((AsyncMaterializationContext) materializationContext).getMtmv());
270+
}
265271
rewrittenPlan = MaterializedViewUtils.rewriteByRules(cascadesContext,
266272
childContext -> {
267273
Rewriter.getWholeTreeRewriter(childContext).execute();
@@ -371,11 +377,11 @@ protected List<Plan> doRewrite(StructInfo queryStructInfo, CascadesContext casca
371377
logicalProperties, queryPlan.getLogicalProperties()));
372378
continue;
373379
}
374-
recordIfRewritten(queryStructInfo.getOriginalPlan(), materializationContext, cascadesContext);
375380
trySetStatistics(materializationContext, cascadesContext);
376381
rewriteResults.add(rewrittenPlan);
377382
// if rewrite successfully, try to regenerate mv scan because it maybe used again
378383
materializationContext.tryReGenerateScanPlan(cascadesContext);
384+
recordIfRewritten(queryStructInfo.getOriginalPlan(), materializationContext, cascadesContext);
379385
}
380386
return rewriteResults;
381387
}

‎fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -260,7 +260,12 @@ public PlanChecker rewrite() {
260260
public PlanChecker optimize() {
261261
cascadesContext.setJobContext(PhysicalProperties.GATHER);
262262
double now = System.currentTimeMillis();
263-
new Optimizer(cascadesContext).execute();
263+
try {
264+
new Optimizer(cascadesContext).execute();
265+
} finally {
266+
// Mv rewrite add lock manually, so need release manually
267+
cascadesContext.getStatementContext().releasePlannerResources();
268+
}
264269
System.out.println("cascades:" + (System.currentTimeMillis() - now));
265270
return this;
266271
}
Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
suite("dml_rewrite_with_lock", "zfr_mtmv_test") {
19+
20+
String db = context.config.getDbNameByFile(context.file)
21+
sql "use ${db}"
22+
sql "SET enable_materialized_view_rewrite=true"
23+
sql "SET enable_materialized_view_nest_rewrite=true"
24+
sql "SET enable_materialized_view_union_rewrite=true"
25+
26+
sql """
27+
drop table if exists lineitem_range_date_union
28+
"""
29+
30+
sql """CREATE TABLE `lineitem_range_date_union` (
31+
`l_orderkey` BIGINT NULL,
32+
`l_linenumber` INT NULL,
33+
`l_partkey` INT NULL,
34+
`l_suppkey` INT NULL,
35+
`l_quantity` DECIMAL(15, 2) NULL,
36+
`l_extendedprice` DECIMAL(15, 2) NULL,
37+
`l_discount` DECIMAL(15, 2) NULL,
38+
`l_tax` DECIMAL(15, 2) NULL,
39+
`l_returnflag` VARCHAR(1) NULL,
40+
`l_linestatus` VARCHAR(1) NULL,
41+
`l_commitdate` DATE NULL,
42+
`l_receiptdate` DATE NULL,
43+
`l_shipinstruct` VARCHAR(25) NULL,
44+
`l_shipmode` VARCHAR(10) NULL,
45+
`l_comment` VARCHAR(44) NULL,
46+
`l_shipdate` DATE not NULL
47+
) ENGINE=OLAP
48+
DUPLICATE KEY(l_orderkey, l_linenumber, l_partkey, l_suppkey )
49+
COMMENT 'OLAP'
50+
partition by range (`l_shipdate`) (
51+
partition p1 values [("2023-10-29"), ("2023-10-30")),
52+
partition p2 values [("2023-10-30"), ("2023-10-31")),
53+
partition p3 values [("2023-10-31"), ("2023-11-01")))
54+
DISTRIBUTED BY HASH(`l_orderkey`) BUCKETS 96
55+
PROPERTIES (
56+
"replication_allocation" = "tag.location.default: 1"
57+
);"""
58+
59+
sql """
60+
drop table if exists orders_range_date_union
61+
"""
62+
63+
sql """CREATE TABLE `orders_range_date_union` (
64+
`o_orderkey` BIGINT NULL,
65+
`o_custkey` INT NULL,
66+
`o_orderstatus` VARCHAR(1) NULL,
67+
`o_totalprice` DECIMAL(15, 2) NULL,
68+
`o_orderpriority` VARCHAR(15) NULL,
69+
`o_clerk` VARCHAR(15) NULL,
70+
`o_shippriority` INT NULL,
71+
`o_comment` VARCHAR(79) NULL,
72+
`o_orderdate` DATE not NULL
73+
) ENGINE=OLAP
74+
DUPLICATE KEY(`o_orderkey`, `o_custkey`)
75+
COMMENT 'OLAP'
76+
partition by range (`o_orderdate`) (
77+
partition p1 values [("2023-10-29"), ("2023-10-30")),
78+
partition p2 values [("2023-10-30"), ("2023-10-31")),
79+
partition p3 values [("2023-10-31"), ("2023-11-01")),
80+
partition p4 values [("2023-11-01"), ("2023-11-02")),
81+
partition p5 values [("2023-11-02"), ("2023-11-03")))
82+
DISTRIBUTED BY HASH(`o_orderkey`) BUCKETS 96
83+
PROPERTIES (
84+
"replication_allocation" = "tag.location.default: 1"
85+
);"""
86+
87+
sql """
88+
insert into lineitem_range_date_union values
89+
(null, 1, 2, 3, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-29'),
90+
(1, null, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-29'),
91+
(3, 3, null, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', '2023-10-19', 'c', 'd', 'xxxxxxxxx', '2023-10-31'),
92+
(1, 2, 3, null, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-29'),
93+
(2, 3, 2, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', null, '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-30'),
94+
(3, 1, 1, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', null, 'c', 'd', 'xxxxxxxxx', '2023-10-31'),
95+
(1, 3, 2, 2, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-29');
96+
"""
97+
98+
sql """
99+
insert into orders_range_date_union values
100+
(null, 1, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-29'),
101+
(1, null, 'o', 109.2, 'c','d',2, 'mm', '2023-10-29'),
102+
(3, 3, null, 99.5, 'a', 'b', 1, 'yy', '2023-10-30'),
103+
(1, 2, 'o', null, 'a', 'b', 1, 'yy', '2023-11-01'),
104+
(2, 3, 'k', 109.2, null,'d',2, 'mm', '2023-11-02'),
105+
(3, 1, 'k', 99.5, 'a', null, 1, 'yy', '2023-11-02'),
106+
(1, 3, 'o', 99.5, 'a', 'b', null, 'yy', '2023-10-31'),
107+
(2, 1, 'o', 109.2, 'c','d',2, null, '2023-10-30'),
108+
(3, 2, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-29'),
109+
(4, 5, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-31');
110+
"""
111+
112+
sql """DROP MATERIALIZED VIEW if exists day_mv;"""
113+
create_async_mv(db, "day_mv",
114+
"""select date_trunc(`l_shipdate`, 'day') as col1, l_shipdate, l_orderkey
115+
from lineitem_range_date_union as t1 left join orders_range_date_union as t2
116+
on t1.l_orderkey = t2.o_orderkey group by col1, l_shipdate, l_orderkey;
117+
"""
118+
)
119+
120+
def query1 = """
121+
select date_trunc(`l_shipdate`, 'day') as col1, l_shipdate, l_orderkey
122+
from lineitem_range_date_union as t1 left join orders_range_date_union as t2
123+
on t1.l_orderkey = t2.o_orderkey
124+
group by col1, l_shipdate, l_orderkey
125+
"""
126+
127+
mv_rewrite_success(query1, "day_mv")
128+
129+
def query2 = """
130+
select date_trunc(`l_shipdate`, 'hour') as col1, l_shipdate, l_orderkey from
131+
lineitem_range_date_union as t1 left join orders_range_date_union as t2
132+
on t1.l_orderkey = t2.o_orderkey
133+
group by col1, l_shipdate, l_orderkey
134+
"""
135+
136+
sql """DROP MATERIALIZED VIEW if exists hour_mv;"""
137+
create_async_mv(db, "hour_mv",
138+
"""
139+
select date_trunc(`l_shipdate`, 'hour') as col1, l_shipdate, l_orderkey from
140+
lineitem_range_date_union as t1 left join orders_range_date_union as t2
141+
on t1.l_orderkey = t2.o_orderkey group by col1, l_shipdate, l_orderkey;
142+
""")
143+
mv_rewrite_success(query2, "hour_mv")
144+
145+
146+
sql """alter table lineitem_range_date_union add partition p4 values [("2023-11-01"), ("2023-11-02"));"""
147+
sql """insert into lineitem_range_date_union values
148+
(1, null, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-11-01')"""
149+
150+
sql """refresh MATERIALIZED VIEW hour_mv auto;"""
151+
waitingMTMVTaskFinishedByMvName("hour_mv")
152+
153+
sql """refresh MATERIALIZED VIEW day_mv auto;"""
154+
waitingMTMVTaskFinishedByMvName("day_mv")
155+
156+
mv_rewrite_success(query1, "day_mv")
157+
mv_rewrite_success(query2, "hour_mv")
158+
}
159+
160+
161+

0 commit comments

Comments
 (0)