Skip to content

Commit 01efbca

Browse files
authored
[fix](ivm) Fall back to complete refresh when the IVM stream is unusable (#68170)
When an IVM incremental refresh hits a missing or unusable base table stream, the rewrite fails with `STREAM_UNSUPPORTED`. Previously this reason fell back to the partition-based refresh, which reads the same broken stream again and fails the task, so the refresh never reached the COMPLETE attempt whose reconcile step recreates the stream. Only an explicit `REFRESH COMPLETE` could recover. Two changes: 1. Add `STREAM_UNSUPPORTED` to `IvmFailureReason.requiresCompleteRefresh()`, so an unusable stream jumps directly to the COMPLETE attempt that reconciles it. 2. Move `reconcileIvmStreams` from the shared partition-based path into `executeCompleteAttempt`. Behavior is unchanged (COMPLETE was its only triggering mode), but it makes the complete attempt the single place that resets stream baselines, which a stream's global baseline requires. IVM incremental refresh now recovers automatically from an unusable base table stream by falling back to a complete refresh instead of failing the task.
1 parent a7b7590 commit 01efbca

8 files changed

Lines changed: 822 additions & 124 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java‎

Lines changed: 200 additions & 44 deletions
Large diffs are not rendered by default.

‎fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmFailureReason.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ public enum IvmFailureReason {
3838

3939
public boolean requiresCompleteRefresh() {
4040
return this == BINLOG_BROKEN
41+
|| this == STREAM_UNSUPPORTED
4142
|| this == MIN_MAX_BOUNDARY_HIT
4243
|| this == BITMAP_AGG_DELETE
4344
|| this == PLAN_SIGNATURE_MISMATCH;

‎fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTaskTest.java‎

Lines changed: 250 additions & 80 deletions
Large diffs are not rendered by default.

‎fe/fe-core/src/test/java/org/apache/doris/mtmv/ivm/IvmFailureReasonTest.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,9 @@ public class IvmFailureReasonTest {
2525
@Test
2626
public void testRequiresCompleteRefresh() {
2727
Assertions.assertTrue(IvmFailureReason.BINLOG_BROKEN.requiresCompleteRefresh());
28+
// An unusable stream can only be recreated by the COMPLETE path's reconcile step; a
29+
// PARTITIONS fallback would read the same stream and fail again.
30+
Assertions.assertTrue(IvmFailureReason.STREAM_UNSUPPORTED.requiresCompleteRefresh());
2831
Assertions.assertTrue(IvmFailureReason.MIN_MAX_BOUNDARY_HIT.requiresCompleteRefresh());
2932
Assertions.assertTrue(IvmFailureReason.BITMAP_AGG_DELETE.requiresCompleteRefresh());
3033
Assertions.assertTrue(IvmFailureReason.PLAN_SIGNATURE_MISMATCH.requiresCompleteRefresh());
Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
-- This file is automatically generated. You should know what you did if you want to edit this
2+
-- !downstream_mv --
3+
1 10
4+
2 20
5+
3 30
6+
Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
-- This file is automatically generated. You should know what you did if you want to edit this
2+
-- !baseline_mv --
3+
1 2026-01-10 100 dim-a
4+
2 2026-02-10 200 dim-b
5+
6+
-- !reconciled_mv --
7+
1 2026-01-10 100 dim-a
8+
2 2026-02-10 200 dim-b
9+
10+
-- !after_strict_failure_mv --
11+
1 2026-01-10 100 dim-a
12+
2 2026-02-10 200 dim-b
13+
Lines changed: 162 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
import org.awaitility.Awaitility
19+
20+
import static java.util.concurrent.TimeUnit.SECONDS
21+
22+
/**
23+
* Which streams decide that a refresh needs the complete attempt.
24+
*
25+
* <p>An IVM MV is created with a stream for every base table in its relation, and for a chained MV the
26+
* relation carries more than the plan reads: mv2 reads mv1, while the table behind mv1 is in mv2's
27+
* relation only through the closure. Both rewrites that read streams -- the incremental delta rewriter
28+
* and the full refresh -- take the streams of the plan's scans, so mv2's stream on that table is never
29+
* read, and its absence must not turn mv2's incremental refresh into a rebuild of the whole MV.
30+
*
31+
* <p>Pinned here:
32+
* <ol>
33+
* <li>the unused stream exists, so the setup is the one the case is about;</li>
34+
* <li>an incremental refresh of the upstream MV succeeds with it dropped, which is what makes the
35+
* refresh below able to stay incremental;</li>
36+
* <li>the downstream MV still refreshes incrementally rather than falling back to a complete
37+
* refresh for a stream nothing reads.</li>
38+
* </ol>
39+
*/
40+
suite("test_ivm_chained_stream_scope") {
41+
def baseTable = "ivm_chained_scope_base"
42+
def upstreamMv = "ivm_chained_scope_up"
43+
def downstreamMv = "ivm_chained_scope_down"
44+
45+
def waitForNewTask = { String mvName, String previousTaskId ->
46+
def taskResult
47+
Awaitility.await().atMost(300, SECONDS).pollInterval(2, SECONDS).until({
48+
taskResult = sql_return_maparray("""
49+
SELECT TaskId, Status
50+
FROM tasks('type'='mv')
51+
WHERE MvDatabaseName = '${context.dbName}'
52+
AND MvName = '${mvName}'
53+
ORDER BY CreateTime DESC, TaskId DESC LIMIT 1
54+
""")
55+
return !taskResult.isEmpty()
56+
&& taskResult[0].TaskId.toString() != previousTaskId
57+
&& taskResult[0].Status.toString() != 'PENDING'
58+
&& taskResult[0].Status.toString() != 'RUNNING'
59+
})
60+
return taskResult[0].TaskId.toString()
61+
}
62+
63+
def taskRecord = { String taskId ->
64+
// Unset RefreshMode comes back as the literal two-character string "\N", which does not survive
65+
// the .out round trip, so fold the unset value into a printable token.
66+
return sql_return_maparray("""
67+
SELECT Status,
68+
CASE WHEN RefreshMode IN ('COMPLETE', 'PARTIAL', 'NOT_REFRESH')
69+
THEN RefreshMode ELSE 'NONE' END AS RefreshMode
70+
FROM tasks('type'='mv')
71+
WHERE TaskId = '${taskId}'
72+
""")[0]
73+
}
74+
75+
def mvId = { String mvName ->
76+
def rows = sql_return_maparray("""
77+
SELECT Id FROM mv_infos('database'='${context.dbName}') WHERE Name = '${mvName}'
78+
""")
79+
return rows[0].Id.toString()
80+
}
81+
82+
// Streams are named after the MV that owns them, so the ones of the downstream MV on the upstream's
83+
// base table can be told apart from the ones the upstream MV has on it.
84+
def streamNamesOf = { String mvName, String baseTableName ->
85+
def rows = sql_return_maparray("""
86+
SELECT STREAM_NAME FROM information_schema.table_streams
87+
WHERE DB_NAME = '${context.dbName}' AND BASE_TABLE_NAME = '${baseTableName}'
88+
""")
89+
def prefix = "__doris_ivm_stream_${mvId(mvName)}_"
90+
return rows.collect { it.STREAM_NAME.toString() }.findAll { it.startsWith(prefix) }
91+
}
92+
93+
sql """DROP MATERIALIZED VIEW IF EXISTS ${downstreamMv}"""
94+
sql """DROP MATERIALIZED VIEW IF EXISTS ${upstreamMv}"""
95+
sql """DROP TABLE IF EXISTS ${baseTable}"""
96+
97+
sql """
98+
CREATE TABLE ${baseTable} (
99+
k1 INT NOT NULL,
100+
v1 INT
101+
)
102+
UNIQUE KEY(k1)
103+
DISTRIBUTED BY HASH(k1) BUCKETS 1
104+
PROPERTIES (
105+
"replication_num" = "1",
106+
"enable_unique_key_merge_on_write" = "true",
107+
"binlog.enable" = "true",
108+
"binlog.format" = "ROW",
109+
"binlog.need_historical_value" = "true"
110+
)
111+
"""
112+
sql """INSERT INTO ${baseTable} VALUES (1, 10), (2, 20)"""
113+
114+
// The upstream MV carries row binlog, which is what the downstream MV reads its changes from.
115+
sql """
116+
CREATE MATERIALIZED VIEW ${upstreamMv}
117+
BUILD DEFERRED REFRESH INCREMENTAL ON MANUAL
118+
DISTRIBUTED BY RANDOM BUCKETS 1
119+
PROPERTIES (
120+
"replication_num" = "1",
121+
"binlog.enable" = "true",
122+
"binlog.format" = "ROW",
123+
"binlog.need_historical_value" = "true"
124+
)
125+
AS SELECT * FROM ${baseTable}
126+
"""
127+
sql """REFRESH MATERIALIZED VIEW ${upstreamMv} COMPLETE"""
128+
def upstreamTaskId = waitForNewTask(upstreamMv, null)
129+
130+
sql """
131+
CREATE MATERIALIZED VIEW ${downstreamMv}
132+
BUILD DEFERRED REFRESH INCREMENTAL ON MANUAL
133+
DISTRIBUTED BY RANDOM BUCKETS 1
134+
PROPERTIES ("replication_num" = "1")
135+
AS SELECT k1, v1 FROM ${upstreamMv}
136+
"""
137+
sql """REFRESH MATERIALIZED VIEW ${downstreamMv} COMPLETE"""
138+
def downstreamTaskId = waitForNewTask(downstreamMv, null)
139+
140+
// The downstream MV's plan scans the upstream MV only, but its relation also carries the upstream's
141+
// base table, and the MV was created with a stream for it.
142+
def unusedStreams = streamNamesOf(downstreamMv, baseTable)
143+
assertEquals(1, unusedStreams.size())
144+
sql """DROP STREAM ${context.dbName}.${unusedStreams[0]} FORCE"""
145+
assertEquals(0, streamNamesOf(downstreamMv, baseTable).size())
146+
147+
// One change reaches the downstream MV through the upstream MV, so the refresh below has something
148+
// to apply and cannot pass as an empty incremental refresh.
149+
sql """INSERT INTO ${baseTable} VALUES (3, 30)"""
150+
sql """REFRESH MATERIALIZED VIEW ${upstreamMv} INCREMENTAL"""
151+
upstreamTaskId = waitForNewTask(upstreamMv, upstreamTaskId)
152+
assertEquals("SUCCESS", taskRecord(upstreamTaskId).Status.toString())
153+
154+
sql """REFRESH MATERIALIZED VIEW ${downstreamMv} AUTO"""
155+
downstreamTaskId = waitForNewTask(downstreamMv, downstreamTaskId)
156+
def downstreamTask = taskRecord(downstreamTaskId)
157+
// An incremental refresh of an IVM MV records no refresh mode, so an unset mode is the incremental
158+
// attempt succeeding; COMPLETE would say the whole MV was rebuilt.
159+
assertEquals("SUCCESS", downstreamTask.Status.toString())
160+
assertEquals("NONE", downstreamTask.RefreshMode.toString())
161+
order_qt_downstream_mv """SELECT k1, v1 FROM ${downstreamMv}"""
162+
}
Lines changed: 187 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,187 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
import org.awaitility.Awaitility
19+
20+
import static java.util.concurrent.TimeUnit.SECONDS
21+
22+
/**
23+
* What a partition-based refresh does when the MV's base table streams are gone.
24+
*
25+
* <p>A partition refresh reads the stream of every base table it realigns from -- the PCT tables of the
26+
* partitions it rebuilds, and the non-PCT tables the MV joins -- so a missing stream makes it fail,
27+
* and only the COMPLETE attempt reconciles streams. A request that allows falling back must therefore
28+
* end up doing COMPLETE rather than failing the task; a request that does not allow it must keep
29+
* failing, because COMPLETE would refresh more than it was asked to.
30+
*
31+
* <p>Driven by dropping the joined table's stream by hand: that is the state the fallback exists for and the one a
32+
* unit test can only mock.
33+
*/
34+
suite("test_ivm_partitions_fallback_stream_unusable") {
35+
def factTable = "ivm_stream_f"
36+
def dimTable = "ivm_stream_d"
37+
def mvName = "ivm_stream_mv"
38+
39+
def waitForNewTask = { previousTaskId ->
40+
def taskResult
41+
Awaitility.await().atMost(300, SECONDS).pollInterval(2, SECONDS).until({
42+
taskResult = sql_return_maparray("""
43+
SELECT TaskId, Status
44+
FROM tasks('type'='mv')
45+
WHERE MvDatabaseName = '${context.dbName}'
46+
AND MvName = '${mvName}'
47+
ORDER BY CreateTime DESC, TaskId DESC LIMIT 1
48+
""")
49+
return !taskResult.isEmpty()
50+
&& taskResult[0].TaskId.toString() != previousTaskId
51+
&& taskResult[0].Status.toString() != 'PENDING'
52+
&& taskResult[0].Status.toString() != 'RUNNING'
53+
})
54+
return taskResult[0].TaskId.toString()
55+
}
56+
57+
// Unset IvmFallbackReason comes back as the literal two-character string "\N", which does not
58+
// survive a .out round trip, so fold it into a printable token.
59+
def taskOutcome = { String taskId ->
60+
def rows = sql_return_maparray("""
61+
SELECT Status,
62+
CASE WHEN RefreshMode IN ('COMPLETE', 'PARTIAL', 'NOT_REFRESH')
63+
THEN RefreshMode ELSE 'NONE' END AS Mode,
64+
CASE WHEN IvmFallbackReason = 'BINLOG_BROKEN' OR IvmFallbackReason = 'STREAM_UNSUPPORTED'
65+
THEN IvmFallbackReason ELSE 'NONE' END AS Fallback
66+
FROM tasks('type'='mv')
67+
WHERE TaskId = '${taskId}'
68+
""")
69+
// toString: a GString never equals a String, and these are compared against literals.
70+
return "${rows[0].Status}\t${rows[0].Mode}\t${rows[0].Fallback}".toString()
71+
}
72+
73+
def taskStatus = { String taskId ->
74+
def rows = sql_return_maparray("""
75+
SELECT Status FROM tasks('type'='mv') WHERE TaskId = '${taskId}'
76+
""")
77+
return rows[0].Status.toString()
78+
}
79+
80+
// The streams of every suite in this directory live in the one database the runner gives them, so
81+
// they have to be looked up by base table: taking every stream of the database would take the
82+
// streams of the suites running next to this one, and fail them with "IVM stream not found".
83+
def streamNameOf = { String tableName ->
84+
def rows = sql("""
85+
SELECT STREAM_NAME FROM information_schema.table_streams
86+
WHERE DB_NAME = '${context.dbName}' AND BASE_TABLE_NAME = '${tableName}'
87+
""")
88+
return rows.isEmpty() ? null : rows[0][0].toString()
89+
}
90+
91+
def dropStreamOf = { String tableName ->
92+
def name = streamNameOf(tableName)
93+
assertTrue(name != null, "the MV should have created a stream for " + tableName)
94+
sql """DROP STREAM ${name} FORCE"""
95+
}
96+
97+
sql """DROP MATERIALIZED VIEW IF EXISTS ${mvName}"""
98+
sql """DROP TABLE IF EXISTS ${factTable}"""
99+
sql """DROP TABLE IF EXISTS ${dimTable}"""
100+
101+
sql """
102+
CREATE TABLE ${factTable} (
103+
order_id BIGINT NOT NULL,
104+
dt DATE NOT NULL,
105+
dimension_id INT,
106+
amount INT
107+
)
108+
UNIQUE KEY(order_id, dt)
109+
PARTITION BY RANGE(dt) ()
110+
DISTRIBUTED BY HASH(order_id) BUCKETS 1
111+
PROPERTIES (
112+
"replication_num" = "1",
113+
"enable_unique_key_merge_on_write" = "true",
114+
"binlog.enable" = "true",
115+
"binlog.format" = "ROW",
116+
"binlog.need_historical_value" = "true"
117+
)
118+
"""
119+
sql """ALTER TABLE ${factTable} ADD PARTITION p202601 VALUES [('2026-01-01'), ('2026-02-01'))"""
120+
sql """ALTER TABLE ${factTable} ADD PARTITION p202602 VALUES [('2026-02-01'), ('2026-03-01'))"""
121+
122+
// Partitioned as well, but joined on a non-partition column: a base table of the MV that is not one
123+
// of its PCT tables, and therefore read through its stream by a partition refresh.
124+
sql """
125+
CREATE TABLE ${dimTable} (
126+
dimension_id INT NOT NULL,
127+
dt DATE NOT NULL,
128+
dimension_name VARCHAR(32)
129+
)
130+
UNIQUE KEY(dimension_id, dt)
131+
PARTITION BY RANGE(dt) ()
132+
DISTRIBUTED BY HASH(dimension_id) BUCKETS 1
133+
PROPERTIES (
134+
"replication_num" = "1",
135+
"enable_unique_key_merge_on_write" = "true",
136+
"binlog.enable" = "true",
137+
"binlog.format" = "ROW",
138+
"binlog.need_historical_value" = "true"
139+
)
140+
"""
141+
sql """ALTER TABLE ${dimTable} ADD PARTITION d202601 VALUES [('2026-01-01'), ('2026-02-01'))"""
142+
sql """ALTER TABLE ${dimTable} ADD PARTITION d202602 VALUES [('2026-02-01'), ('2026-03-01'))"""
143+
144+
sql """INSERT INTO ${dimTable} VALUES (10, '2026-01-15', 'dim-a'), (20, '2026-02-15', 'dim-b')"""
145+
sql """INSERT INTO ${factTable} VALUES
146+
(1, '2026-01-10', 10, 100),
147+
(2, '2026-02-10', 20, 200)"""
148+
149+
sql """
150+
CREATE MATERIALIZED VIEW ${mvName}
151+
BUILD DEFERRED REFRESH INCREMENTAL ON MANUAL
152+
KEY(order_id, dt)
153+
PARTITION BY(dt)
154+
DISTRIBUTED BY HASH(order_id) BUCKETS 1
155+
PROPERTIES ("replication_num" = "1")
156+
AS SELECT f.order_id, f.dt, f.amount, d.dimension_name
157+
FROM ${factTable} f
158+
LEFT JOIN ${dimTable} d ON f.dimension_id = d.dimension_id
159+
"""
160+
161+
sql """REFRESH MATERIALIZED VIEW ${mvName} COMPLETE"""
162+
def taskId = waitForNewTask(null)
163+
assertEquals("SUCCESS\tCOMPLETE\tNONE", taskOutcome(taskId))
164+
order_qt_baseline_mv """SELECT order_id, dt, amount, dimension_name FROM ${mvName}"""
165+
166+
// With the streams gone, a partition refresh cannot read anything. Falling back is what makes it
167+
// reach the COMPLETE attempt, which reconciles the streams and rebuilds the MV; without it the
168+
// refresh would fail and stay failed.
169+
dropStreamOf(dimTable)
170+
sql """REFRESH MATERIALIZED VIEW ${mvName} PARTITIONS FALLBACK"""
171+
taskId = waitForNewTask(taskId)
172+
assertEquals("SUCCESS\tCOMPLETE\tSTREAM_UNSUPPORTED", taskOutcome(taskId))
173+
assertTrue(streamNameOf(dimTable) != null, "the complete attempt should have reconciled the stream")
174+
order_qt_reconciled_mv """SELECT order_id, dt, amount, dimension_name FROM ${mvName}"""
175+
176+
// The same request without fallback keeps failing: COMPLETE refreshes more than the request asked
177+
// for, so it may only be reached when the user allowed it. Its failure leaves the MV as it was.
178+
//
179+
// The base table has to be out of sync for the partition refresh to do anything at all: an MV that
180+
// is already up to date refreshes no partition, and therefore never reads a stream.
181+
sql """INSERT INTO ${factTable} VALUES (3, '2026-01-20', 10, 300)"""
182+
dropStreamOf(dimTable)
183+
sql """REFRESH MATERIALIZED VIEW ${mvName} PARTITIONS"""
184+
taskId = waitForNewTask(taskId)
185+
assertEquals("FAILED", taskStatus(taskId))
186+
order_qt_after_strict_failure_mv """SELECT order_id, dt, amount, dimension_name FROM ${mvName}"""
187+
}

0 commit comments

Comments
 (0)