Skip to content

Commit 047882e

Browse files
committed
[fix](fe) Install cloud table stream read states for IVM refresh
IVM refresh synthesizes LogicalOlapTableStreamScan after relation binding: COMPLETE rewrites base scans in IvmFullRefreshMTMV and INCREMENTAL creates delta scans via IvmDeltaRewriteState. These scans never pass BindRelation, so CloudTableStreamReadStateHook is never registered for their statements and their cloud read states are never installed. Since apache#66739 tightened StreamConsumptionInfoExtractor, every IVM MV refresh failed in cloud mode with "Cloud Table Stream read state must be installed during relation analysis". Key changes: - Expose the install orchestration as public static CloudTableStreamReadStateHook.installReadStates(Plan): no-op on shared-nothing and on plans without stream scans - Register the hook in IvmFullRefreshMTMV so afterAnalyze batch-installs read states for COMPLETE refresh stream scans - Call installReadStates at the single return of IvmIncrRefreshMTMV.rewriteRoot for INCREMENTAL delta scans - Update the extractor error message accordingly Unit Test: - run-regression-test.sh -d mtmv_p0/ivm -s test_ivm_basic_mtmv passed on a local cloud cluster (1FE+1BE+1MS+1Recycler+FDB), previously failing
1 parent 06a7eac commit 047882e

4 files changed

Lines changed: 30 additions & 8 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CloudTableStreamReadStateHook.java‎

Lines changed: 23 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import org.apache.doris.catalog.stream.OlapTableStreamWrapper;
2121
import org.apache.doris.cloud.proto.Cloud;
2222
import org.apache.doris.cloud.rpc.CloudTableStreamReadStateHelper;
23+
import org.apache.doris.common.Config;
2324
import org.apache.doris.common.UserException;
2425
import org.apache.doris.nereids.NereidsPlanner;
2526
import org.apache.doris.nereids.PlannerHook;
@@ -38,7 +39,13 @@
3839
import java.util.Map;
3940
import java.util.Set;
4041

41-
/** Resolves one read snapshot for every Cloud Table Stream relation after statement analysis. */
42+
/**
43+
* Resolves one read snapshot for every Cloud Table Stream relation after statement analysis.
44+
*
45+
* <p>Also serves as the public installation entry for statements that synthesize stream scans
46+
* outside relation binding (for example refresh flows that rewrite base-table scans into stream
47+
* scans after binding): call {@link #installReadStates(Plan)} once the final plan is ready.
48+
*/
4249
public class CloudTableStreamReadStateHook implements PlannerHook {
4350
public static final CloudTableStreamReadStateHook INSTANCE = new CloudTableStreamReadStateHook();
4451

@@ -47,15 +54,26 @@ private CloudTableStreamReadStateHook() {
4754

4855
@Override
4956
public void afterAnalyze(NereidsPlanner planner) {
50-
resolve(planner.getCascadesContext().getRewritePlan());
57+
installReadStates(planner.getCascadesContext().getRewritePlan());
5158
}
5259

53-
static void resolve(Plan plan) {
60+
/**
61+
* Installs one read snapshot for every Cloud Table Stream scan in the given plan.
62+
*
63+
* <p>Public so that statements generating stream scans outside relation binding can pin their
64+
* read states before extraction. No-op when the plan has no stream scans or when every scan
65+
* already carries installed states.
66+
*/
67+
public static void installReadStates(Plan plan) {
68+
if (Config.isNotCloudMode()) {
69+
return;
70+
}
5471
List<LogicalOlapTableStreamScan> scans = new ArrayList<>();
5572
plan.collectToList(LogicalOlapTableStreamScan.class::isInstance).forEach(node ->
5673
scans.add((LogicalOlapTableStreamScan) node));
57-
Preconditions.checkState(!scans.isEmpty(),
58-
"Cloud Table Stream read-state hook requires at least one Stream scan");
74+
if (scans.isEmpty()) {
75+
return;
76+
}
5977

6078
boolean readStatesInstalled = scans.get(0).getTable().hasCloudReadStates();
6179
for (LogicalOlapTableStreamScan scan : scans) {

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/IvmFullRefreshMTMV.java‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -66,8 +66,11 @@ public Rule build() {
6666
.filter(context -> context.getMode() == IvmRewriteContext.Mode.FULL)
6767
.filter(IvmRewriteContext::hasFullRefreshStreamScans)
6868
.isPresent())
69-
.thenApply(ctx -> rewriteScan(ctx.root,
70-
ctx.statementContext.getIvmRewriteContext().get()))
69+
.thenApply(ctx -> {
70+
ctx.statementContext.addPlannerHook(CloudTableStreamReadStateHook.INSTANCE);
71+
return rewriteScan(ctx.root,
72+
ctx.statementContext.getIvmRewriteContext().get());
73+
})
7174
.toRule(RuleType.IVM_FULL_REFRESH_MTMV);
7275
}
7376

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/IvmIncrRefreshMTMV.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ public Plan rewriteRoot(Plan plan, JobContext jobContext) {
6464
IvmRewriteContext context = rewriteContext.get();
6565
Plan rewritten = rewriteIncrementalPlan(plan, rewriteResult, context,
6666
jobContext.getCascadesContext().getConnectContext());
67+
CloudTableStreamReadStateHook.installReadStates(rewritten);
6768
rewriteResult.setIncrRefreshRewritten(true);
6869
return rewritten;
6970
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/StreamConsumptionInfoExtractor.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ public static List<TableStreamUpdateInfo> extract(Plan analyzedPlan) {
5757
if (!streamScan.isSnapshot()) {
5858
OlapTableStreamWrapper wrapper = streamScan.getTable();
5959
Preconditions.checkState(Config.isNotCloudMode() || wrapper.hasCloudReadStates(),
60-
"Cloud Table Stream read state must be installed during relation analysis");
60+
"Cloud Table Stream read state must be installed before extraction");
6161
AbstractTableStreamUpdate update = Config.isCloudMode()
6262
? toCloudOlapTableStreamUpdate(wrapper, streamScan.getSelectedPartitionIds())
6363
: toOlapTableStreamUpdate(wrapper);

0 commit comments

Comments
 (0)