|
| 1 | +// Licensed to the Apache Software Foundation (ASF) under one |
| 2 | +// or more contributor license agreements. See the NOTICE file |
| 3 | +// distributed with this work for additional information |
| 4 | +// regarding copyright ownership. The ASF licenses this file |
| 5 | +// to you under the Apache License, Version 2.0 (the |
| 6 | +// "License"); you may not use this file except in compliance |
| 7 | +// with the License. You may obtain a copy of the License at |
| 8 | +// |
| 9 | +// http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | +// |
| 11 | +// Unless required by applicable law or agreed to in writing, |
| 12 | +// software distributed under the License is distributed on an |
| 13 | +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 14 | +// KIND, either express or implied. See the License for the |
| 15 | +// specific language governing permissions and limitations |
| 16 | +// under the License. |
| 17 | + |
| 18 | +import org.awaitility.Awaitility |
| 19 | + |
| 20 | +import static java.util.concurrent.TimeUnit.SECONDS |
| 21 | + |
| 22 | +/** |
| 23 | + * What a partition-based refresh does when the MV's base table streams are gone. |
| 24 | + * |
| 25 | + * <p>A partition refresh reads the stream of every base table it realigns from -- the PCT tables of the |
| 26 | + * partitions it rebuilds, and the non-PCT tables the MV joins -- so a missing stream makes it fail, |
| 27 | + * and only the COMPLETE attempt reconciles streams. A request that allows falling back must therefore |
| 28 | + * end up doing COMPLETE rather than failing the task; a request that does not allow it must keep |
| 29 | + * failing, because COMPLETE would refresh more than it was asked to. |
| 30 | + * |
| 31 | + * <p>Driven by dropping the joined table's stream by hand: that is the state the fallback exists for and the one a |
| 32 | + * unit test can only mock. |
| 33 | + */ |
| 34 | +suite("test_ivm_partitions_fallback_stream_unusable") { |
| 35 | + def factTable = "ivm_stream_f" |
| 36 | + def dimTable = "ivm_stream_d" |
| 37 | + def mvName = "ivm_stream_mv" |
| 38 | + |
| 39 | + def waitForNewTask = { previousTaskId -> |
| 40 | + def taskResult |
| 41 | + Awaitility.await().atMost(300, SECONDS).pollInterval(2, SECONDS).until({ |
| 42 | + taskResult = sql_return_maparray(""" |
| 43 | + SELECT TaskId, Status |
| 44 | + FROM tasks('type'='mv') |
| 45 | + WHERE MvDatabaseName = '${context.dbName}' |
| 46 | + AND MvName = '${mvName}' |
| 47 | + ORDER BY CreateTime DESC, TaskId DESC LIMIT 1 |
| 48 | + """) |
| 49 | + return !taskResult.isEmpty() |
| 50 | + && taskResult[0].TaskId.toString() != previousTaskId |
| 51 | + && taskResult[0].Status.toString() != 'PENDING' |
| 52 | + && taskResult[0].Status.toString() != 'RUNNING' |
| 53 | + }) |
| 54 | + return taskResult[0].TaskId.toString() |
| 55 | + } |
| 56 | + |
| 57 | + // Unset IvmFallbackReason comes back as the literal two-character string "\N", which does not |
| 58 | + // survive a .out round trip, so fold it into a printable token. |
| 59 | + def taskOutcome = { String taskId -> |
| 60 | + def rows = sql_return_maparray(""" |
| 61 | + SELECT Status, |
| 62 | + CASE WHEN RefreshMode IN ('COMPLETE', 'PARTIAL', 'NOT_REFRESH') |
| 63 | + THEN RefreshMode ELSE 'NONE' END AS Mode, |
| 64 | + CASE WHEN IvmFallbackReason = 'BINLOG_BROKEN' OR IvmFallbackReason = 'STREAM_UNSUPPORTED' |
| 65 | + THEN IvmFallbackReason ELSE 'NONE' END AS Fallback |
| 66 | + FROM tasks('type'='mv') |
| 67 | + WHERE TaskId = '${taskId}' |
| 68 | + """) |
| 69 | + // toString: a GString never equals a String, and these are compared against literals. |
| 70 | + return "${rows[0].Status}\t${rows[0].Mode}\t${rows[0].Fallback}".toString() |
| 71 | + } |
| 72 | + |
| 73 | + def taskStatus = { String taskId -> |
| 74 | + def rows = sql_return_maparray(""" |
| 75 | + SELECT Status FROM tasks('type'='mv') WHERE TaskId = '${taskId}' |
| 76 | + """) |
| 77 | + return rows[0].Status.toString() |
| 78 | + } |
| 79 | + |
| 80 | + // The streams of every suite in this directory live in the one database the runner gives them, so |
| 81 | + // they have to be looked up by base table: taking every stream of the database would take the |
| 82 | + // streams of the suites running next to this one, and fail them with "IVM stream not found". |
| 83 | + def streamNameOf = { String tableName -> |
| 84 | + def rows = sql(""" |
| 85 | + SELECT STREAM_NAME FROM information_schema.table_streams |
| 86 | + WHERE DB_NAME = '${context.dbName}' AND BASE_TABLE_NAME = '${tableName}' |
| 87 | + """) |
| 88 | + return rows.isEmpty() ? null : rows[0][0].toString() |
| 89 | + } |
| 90 | + |
| 91 | + def dropStreamOf = { String tableName -> |
| 92 | + def name = streamNameOf(tableName) |
| 93 | + assertTrue(name != null, "the MV should have created a stream for " + tableName) |
| 94 | + sql """DROP STREAM ${name} FORCE""" |
| 95 | + } |
| 96 | + |
| 97 | + sql """DROP MATERIALIZED VIEW IF EXISTS ${mvName}""" |
| 98 | + sql """DROP TABLE IF EXISTS ${factTable}""" |
| 99 | + sql """DROP TABLE IF EXISTS ${dimTable}""" |
| 100 | + |
| 101 | + sql """ |
| 102 | + CREATE TABLE ${factTable} ( |
| 103 | + order_id BIGINT NOT NULL, |
| 104 | + dt DATE NOT NULL, |
| 105 | + dimension_id INT, |
| 106 | + amount INT |
| 107 | + ) |
| 108 | + UNIQUE KEY(order_id, dt) |
| 109 | + PARTITION BY RANGE(dt) () |
| 110 | + DISTRIBUTED BY HASH(order_id) BUCKETS 1 |
| 111 | + PROPERTIES ( |
| 112 | + "replication_num" = "1", |
| 113 | + "enable_unique_key_merge_on_write" = "true", |
| 114 | + "binlog.enable" = "true", |
| 115 | + "binlog.format" = "ROW", |
| 116 | + "binlog.need_historical_value" = "true" |
| 117 | + ) |
| 118 | + """ |
| 119 | + sql """ALTER TABLE ${factTable} ADD PARTITION p202601 VALUES [('2026-01-01'), ('2026-02-01'))""" |
| 120 | + sql """ALTER TABLE ${factTable} ADD PARTITION p202602 VALUES [('2026-02-01'), ('2026-03-01'))""" |
| 121 | + |
| 122 | + // Partitioned as well, but joined on a non-partition column: a base table of the MV that is not one |
| 123 | + // of its PCT tables, and therefore read through its stream by a partition refresh. |
| 124 | + sql """ |
| 125 | + CREATE TABLE ${dimTable} ( |
| 126 | + dimension_id INT NOT NULL, |
| 127 | + dt DATE NOT NULL, |
| 128 | + dimension_name VARCHAR(32) |
| 129 | + ) |
| 130 | + UNIQUE KEY(dimension_id, dt) |
| 131 | + PARTITION BY RANGE(dt) () |
| 132 | + DISTRIBUTED BY HASH(dimension_id) BUCKETS 1 |
| 133 | + PROPERTIES ( |
| 134 | + "replication_num" = "1", |
| 135 | + "enable_unique_key_merge_on_write" = "true", |
| 136 | + "binlog.enable" = "true", |
| 137 | + "binlog.format" = "ROW", |
| 138 | + "binlog.need_historical_value" = "true" |
| 139 | + ) |
| 140 | + """ |
| 141 | + sql """ALTER TABLE ${dimTable} ADD PARTITION d202601 VALUES [('2026-01-01'), ('2026-02-01'))""" |
| 142 | + sql """ALTER TABLE ${dimTable} ADD PARTITION d202602 VALUES [('2026-02-01'), ('2026-03-01'))""" |
| 143 | + |
| 144 | + sql """INSERT INTO ${dimTable} VALUES (10, '2026-01-15', 'dim-a'), (20, '2026-02-15', 'dim-b')""" |
| 145 | + sql """INSERT INTO ${factTable} VALUES |
| 146 | + (1, '2026-01-10', 10, 100), |
| 147 | + (2, '2026-02-10', 20, 200)""" |
| 148 | + |
| 149 | + sql """ |
| 150 | + CREATE MATERIALIZED VIEW ${mvName} |
| 151 | + BUILD DEFERRED REFRESH INCREMENTAL ON MANUAL |
| 152 | + KEY(order_id, dt) |
| 153 | + PARTITION BY(dt) |
| 154 | + DISTRIBUTED BY HASH(order_id) BUCKETS 1 |
| 155 | + PROPERTIES ("replication_num" = "1") |
| 156 | + AS SELECT f.order_id, f.dt, f.amount, d.dimension_name |
| 157 | + FROM ${factTable} f |
| 158 | + LEFT JOIN ${dimTable} d ON f.dimension_id = d.dimension_id |
| 159 | + """ |
| 160 | + |
| 161 | + sql """REFRESH MATERIALIZED VIEW ${mvName} COMPLETE""" |
| 162 | + def taskId = waitForNewTask(null) |
| 163 | + assertEquals("SUCCESS\tCOMPLETE\tNONE", taskOutcome(taskId)) |
| 164 | + order_qt_baseline_mv """SELECT order_id, dt, amount, dimension_name FROM ${mvName}""" |
| 165 | + |
| 166 | + // With the streams gone, a partition refresh cannot read anything. Falling back is what makes it |
| 167 | + // reach the COMPLETE attempt, which reconciles the streams and rebuilds the MV; without it the |
| 168 | + // refresh would fail and stay failed. |
| 169 | + dropStreamOf(dimTable) |
| 170 | + sql """REFRESH MATERIALIZED VIEW ${mvName} PARTITIONS FALLBACK""" |
| 171 | + taskId = waitForNewTask(taskId) |
| 172 | + assertEquals("SUCCESS\tCOMPLETE\tSTREAM_UNSUPPORTED", taskOutcome(taskId)) |
| 173 | + assertTrue(streamNameOf(dimTable) != null, "the complete attempt should have reconciled the stream") |
| 174 | + order_qt_reconciled_mv """SELECT order_id, dt, amount, dimension_name FROM ${mvName}""" |
| 175 | + |
| 176 | + // The same request without fallback keeps failing: COMPLETE refreshes more than the request asked |
| 177 | + // for, so it may only be reached when the user allowed it. Its failure leaves the MV as it was. |
| 178 | + // |
| 179 | + // The base table has to be out of sync for the partition refresh to do anything at all: an MV that |
| 180 | + // is already up to date refreshes no partition, and therefore never reads a stream. |
| 181 | + sql """INSERT INTO ${factTable} VALUES (3, '2026-01-20', 10, 300)""" |
| 182 | + dropStreamOf(dimTable) |
| 183 | + sql """REFRESH MATERIALIZED VIEW ${mvName} PARTITIONS""" |
| 184 | + taskId = waitForNewTask(taskId) |
| 185 | + assertEquals("FAILED", taskStatus(taskId)) |
| 186 | + order_qt_after_strict_failure_mv """SELECT order_id, dt, amount, dimension_name FROM ${mvName}""" |
| 187 | +} |
0 commit comments