Skip to content

Commit f36ce2d

Browse files
seawindeyujun777
authored andcommitted
[fix](fe) Preserve IVM fallback stream offsets (apache#23)
IVM `PARTITIONS` and `COMPLETE` fallback refreshes rebuilt MV data without binding the full-image read boundary to the stream offset committed by the insert transaction. A concurrent base-table write could therefore make the rebuilt image and committed offset diverge. Multi-batch refreshes could also publish a non-PCT offset before every affected MV partition was rebuilt, while an old `COMPLETE` task could clear a newer binlog-broken event. Root cause: `MTMVTask` captured refresh versions separately from live stream updates, and RESET/SNAPSHOT planning did not carry one immutable `prev/endTso` boundary through scan planning and transaction commit. ```mermaid flowchart LR A[MTMVTask capture] -->|versions, TSO, prev/next| B[StatementContext] B --> C[RESET or SNAPSHOT scan] C --> D[MV insert transaction] D -->|RESET only| E[Commit stream next] A --> F[COMPLETE broken generation] F --> G[Publish baseline after all batches] ``` Key changes: | Area | Change | | --- | --- | | Refresh orchestration | Capture PCT boundaries per batch and non-PCT boundaries once per task; promote PARTIAL to COMPLETE before writing when non-PCT lag exists. | | Scan planning | Rewrite only captured fallback base scans to RESET/SNAPSHOT and rebuild both modes from the same captured image. | | Offset commit | Commit PCT offsets per covered batch and non-PCT offsets only in the final COMPLETE batch. | | Baseline lifecycle | Keep COMPLETE baselines broken until all batches succeed and reject stale finalization with an in-process generation. | | Tests | Add UNIQUE MOW and DUPLICATE multi-batch fallback coverage; align the latest base branch alias test with the current `IvmRewriteResult` API. | The implementation reuses existing stream scans and insert transaction updates. It adds no SQL syntax, configuration, persistent metadata schema, or second offset lifecycle. Known excluded-table and multi-base-table delta planner cases remain separate TODOs.
1 parent e5d5610 commit f36ce2d

22 files changed

Lines changed: 1538 additions & 223 deletions

‎fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -512,6 +512,7 @@ public boolean markIvmBinlogBroken() {
512512
if (ivmInfo == null) {
513513
ivmInfo = new IvmInfo();
514514
}
515+
ivmInfo.increaseBinlogBrokenGeneration();
515516
if (ivmInfo.isBinlogBroken()) {
516517
return false;
517518
}
@@ -522,6 +523,15 @@ public boolean markIvmBinlogBroken() {
522523
}
523524
}
524525

526+
public long getIvmBinlogBrokenGeneration() {
527+
readMvLock();
528+
try {
529+
return ivmInfo == null ? 0 : ivmInfo.getBinlogBrokenGeneration();
530+
} finally {
531+
readMvUnlock();
532+
}
533+
}
534+
525535
public List<String> getInsertedColumnNames() {
526536
List<Column> columns = getBaseSchema(true);
527537
List<String> columnNames = Lists.newArrayListWithExpectedSize(columns.size());

‎fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java‎

Lines changed: 72 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,11 @@
2222
import org.apache.doris.catalog.DatabaseIf;
2323
import org.apache.doris.catalog.Env;
2424
import org.apache.doris.catalog.MTMV;
25+
import org.apache.doris.catalog.OlapTable;
2526
import org.apache.doris.catalog.ScalarType;
2627
import org.apache.doris.catalog.TableIf;
2728
import org.apache.doris.catalog.info.TableNameInfo;
29+
import org.apache.doris.catalog.stream.StreamReadMode;
2830
import org.apache.doris.cloud.qe.ComputeGroupException;
2931
import org.apache.doris.common.AnalysisException;
3032
import org.apache.doris.common.Config;
@@ -48,7 +50,6 @@
4850
import org.apache.doris.metric.MetricRepo;
4951
import org.apache.doris.mtmv.BaseColInfo;
5052
import org.apache.doris.mtmv.BaseTableInfo;
51-
import org.apache.doris.mtmv.MTMVAnalyzeQueryInfo;
5253
import org.apache.doris.mtmv.MTMVBaseTableIf;
5354
import org.apache.doris.mtmv.MTMVPartitionInfo.MTMVPartitionType;
5455
import org.apache.doris.mtmv.MTMVPartitionUtil;
@@ -236,11 +237,8 @@ private PartitionPlanningException(String message, Throwable cause) {
236237
private String ivmFallbackReason;
237238
@SerializedName("cg")
238239
private String computeGroup;
239-
// Temporarily keeps the compact current layout signature hash from the failed IVM probe.
240-
// After the fallback full refresh succeeds, this hash becomes the next incremental baseline.
241-
private String ivmFallbackPlanSignature;
242-
// Runtime-only diagnostic layout string for logging the next baseline after fallback full refresh.
243-
private String ivmFallbackPlanCanonicalString;
240+
// Runtime-only layout signature from the incremental plan that triggered fallback.
241+
private IvmPlanSignature ivmFallbackPlanSignature;
244242

245243
private MTMV mtmv;
246244
private MTMVRelation relation;
@@ -503,13 +501,24 @@ private AttemptResultType executeIvmAttempt(MTMVRefreshContext refreshContext,
503501
throw new JobException("Cannot use " + request.refreshMode
504502
+ " refresh on a materialized view without INCREMENTAL capability.");
505503
}
504+
// A strict INCREMENTAL request must reach IVM so the stream can establish its initial baseline.
505+
// Only fallback-enabled requests may rebuild a missing or invalidated snapshot with COMPLETE.
506+
if (!mtmv.hasRefreshSnapshot() && request.allowFallback) {
507+
ivmFallbackReason = "INCOMPLETE_REFRESH_SNAPSHOT";
508+
LOG.warn("IVM refresh fell back for mv={}, reason=INCOMPLETE_REFRESH_SNAPSHOT, taskId={}. "
509+
+ "Continuing with COMPLETE refresh.", mtmv.getName(), getTaskId());
510+
return AttemptResultType.FALLBACK_TO_COMPLETE;
511+
}
506512
this.completedPartitions = Lists.newCopyOnWriteArrayList();
507513
this.partitionSnapshots = Maps.newConcurrentMap();
508514
ivmFallbackPlanSignature = null;
509-
ivmFallbackPlanCanonicalString = null;
510515
// Determine which partitions need refresh, same as partition-based flow.
511516
this.needRefreshPartitions = MTMVPartitionUtil.getMTMVNeedRefreshPartitions(refreshContext,
512517
relation.getBaseTablesOneLevelAndFromView());
518+
if (mtmv.getIvmInfo().isBinlogBroken()) {
519+
return handleIvmFallbackResult(IvmRefreshResult.fallback(
520+
IvmFailureReason.BINLOG_BROKEN, "Stream binlog is marked as broken"), request);
521+
}
513522
if (CollectionUtils.isEmpty(needRefreshPartitions)) {
514523
LOG.info("IVM incremental refresh skipped for mv={}: all partitions are synced, taskId={}",
515524
mtmv.getName(), getTaskId());
@@ -549,15 +558,27 @@ private AttemptResultType executeIvmAttempt(MTMVRefreshContext refreshContext,
549558
mtmv.getName(), getTaskId());
550559
return AttemptResultType.SUCCESS;
551560
}
561+
return handleIvmFallbackResult(ivmResult, request);
562+
}
563+
564+
private AttemptResultType handleIvmFallbackResult(IvmRefreshResult ivmResult, RefreshRequest request)
565+
throws JobException {
552566
ivmFallbackReason = ivmResult.getFailureReason().name();
567+
ivmFallbackPlanSignature = ivmResult.getPlanSignature();
553568
if (!request.allowFallback) {
554569
throw new JobException(
555570
"IVM incremental refresh failed for mv=" + mtmv.getName()
556571
+ ", reason=" + ivmResult.getFailureReason()
557572
+ ", detail=" + ivmResult.getDetailMessage());
558573
}
559-
// TODO(IVM): More pre-execution failures may require direct COMPLETE
560-
// recovery, such as signature mismatch or invalid binlog state.
574+
if (ivmResult.getFailureReason() == IvmFailureReason.BINLOG_BROKEN) {
575+
// The previous task already entered the IVM execution phase. If
576+
// fallback is allowed, jump directly to COMPLETE recovery instead of
577+
// trying PARTITIONS first.
578+
LOG.warn("IVM previous run incomplete for mv={}, taskId={}. Continuing with COMPLETE recovery.",
579+
mtmv.getName(), getTaskId());
580+
return AttemptResultType.FALLBACK_TO_COMPLETE;
581+
}
561582
if (ivmResult.getFailureReason() == IvmFailureReason.PLAN_SIGNATURE_MISMATCH) {
562583
LOG.warn("IVM refresh fell back for mv={}, reason={}, detail={}, taskId={}. "
563584
+ "Continuing with COMPLETE refresh.",
@@ -594,83 +615,82 @@ private boolean executePartitionBasedRefresh(MTMVRefreshContext refreshContext,
594615

595616
private void executePartitionBasedRefresh(MTMVRefreshContext context)
596617
throws JobException, AnalysisException {
618+
boolean useIvmFallbackStreams = mtmv.isIvm();
597619
Map<TableIf, String> tableWithPartKey = getIncrementalTableMap();
620+
long baselineGeneration = useIvmFallbackStreams
621+
? IvmRefreshManager.markIvmBaselineBroken(mtmv) : -1;
598622
this.completedPartitions = Lists.newCopyOnWriteArrayList();
599623
int refreshPartitionNum = mtmv.getRefreshPartitionNum();
600624
long execNum = (needRefreshPartitions.size() / refreshPartitionNum) + ((needRefreshPartitions.size()
601625
% refreshPartitionNum) > 0 ? 1 : 0);
626+
boolean refreshAllPartitions = Sets.newHashSet(needRefreshPartitions).equals(mtmv.getPartitionNames());
602627
this.partitionSnapshots = Maps.newConcurrentMap();
603628
for (int i = 0; i < execNum; i++) {
604629
int start = i * refreshPartitionNum;
605630
int end = start + refreshPartitionNum;
606631
Set<String> execPartitionNames = Sets.newHashSet(needRefreshPartitions
607632
.subList(start, Math.min(end, needRefreshPartitions.size())));
633+
Map<BaseTableInfo, Set<Long>> batchResetPartitionIds = useIvmFallbackStreams
634+
? collectPctResetPartitionIds(context, execPartitionNames) : Maps.newHashMap();
635+
Optional<IvmRewriteContext> rewriteContext = Optional.empty();
636+
if (useIvmFallbackStreams) {
637+
StreamReadMode nonPctReadMode = i == 0 && refreshAllPartitions
638+
? StreamReadMode.RESET : StreamReadMode.SNAPSHOT;
639+
rewriteContext = Optional.of(
640+
IvmRewriteContext.full(mtmv, batchResetPartitionIds, nonPctReadMode));
641+
}
608642
// need get names before exec
609643
Map<String, MTMVRefreshPartitionSnapshot> execPartitionSnapshots = MTMVPartitionUtil
610644
.generatePartitionSnapshots(context, relation.getBaseTablesOneLevelAndFromView(),
611645
execPartitionNames);
612646
try {
613-
// TODO(IVM): When IVM full refresh falls back here, the refresh SQL should
614-
// bind to a specific TSO snapshot to guarantee that consumedTso exactly matches
615-
// the version the SQL actually reads. Currently TSO support is incomplete, so
616-
// the full refresh reads the latest visible version without TSO binding, so
617-
// ivmPreRefreshTsos capture/reset stays disabled until TSO-bound reads are
618-
// implemented.
619-
executeWithRetry(execPartitionNames, tableWithPartKey);
647+
executeWithRetry(execPartitionNames, tableWithPartKey, rewriteContext);
620648
} catch (Exception e) {
621649
LOG.error("Execution failed after retries: {}", e.getMessage());
622650
throw new JobException(e.getMessage(), e);
623651
}
624652
completedPartitions.addAll(execPartitionNames);
625653
partitionSnapshots.putAll(execPartitionSnapshots);
626654
}
627-
if (mtmv.isIvm()) {
628-
updateIvmPlanSignatureAfterFullRefreshIfNeeded();
655+
if (useIvmFallbackStreams) {
656+
IvmRefreshManager.finishIvmFullRefresh(mtmv, baselineGeneration,
657+
ivmFallbackPlanSignature);
658+
ivmFallbackPlanSignature = null;
629659
}
630660
LOG.info("MTMVTask refresh used snapshot: {}, mvDbName: {}, mvName: {}, taskId: {}", partitionSnapshots,
631661
mtmv.getDatabase().getFullName(), mtmv.getName(), getTaskId());
632662
}
633663

634-
private void updateIvmPlanSignatureAfterFullRefreshIfNeeded() throws JobException {
635-
if (IvmFailureReason.PLAN_SIGNATURE_MISMATCH.name().equals(ivmFallbackReason)) {
636-
refreshIvmPlanSignatureAfterFullRefresh();
637-
return;
638-
}
639-
if (ivmFallbackPlanSignature == null) {
640-
return;
641-
}
642-
IvmRefreshManager.updatePlanSignatureAfterFullRefresh(
643-
mtmv, ivmFallbackPlanSignature, ivmFallbackPlanCanonicalString);
644-
ivmFallbackPlanSignature = null;
645-
ivmFallbackPlanCanonicalString = null;
646-
}
647-
648-
private void refreshIvmPlanSignatureAfterFullRefresh() throws JobException {
649-
try {
650-
ConnectContext ctx = MTMVPlanUtil.createMTMVContext(
651-
mtmv, MTMVPlanUtil.DISABLE_RULES_WHEN_RUN_MTMV_TASK);
652-
MTMVAnalyzeQueryInfo analyzeQueryInfo = MTMVPlanUtil.analyzeQueryWithSql(mtmv, ctx,
653-
Optional.of(IvmRewriteContext.normalize(mtmv)));
654-
IvmPlanSignature currentSignature = analyzeQueryInfo.getIvmRewriteResult().getPlanSignature();
655-
IvmRefreshManager.updatePlanSignatureAfterFullRefresh(
656-
mtmv, currentSignature.getSha256(), currentSignature.getCanonicalString());
657-
ivmFallbackPlanSignature = null;
658-
ivmFallbackPlanCanonicalString = null;
659-
} catch (Exception e) {
660-
throw new JobException("Failed to rebuild IVM plan signature after full refresh, mv="
661-
+ mtmv.getName() + ", detail=" + e.getMessage(), e);
664+
private Map<BaseTableInfo, Set<Long>> collectPctResetPartitionIds(MTMVRefreshContext context,
665+
Set<String> execPartitionNames) throws AnalysisException {
666+
Map<BaseTableInfo, Set<Long>> resetPartitionIds = Maps.newHashMap();
667+
for (String mvPartitionName : execPartitionNames) {
668+
for (Entry<MTMVRelatedTableIf, Set<String>> entry
669+
: context.getByPartitionName(mvPartitionName).entrySet()) {
670+
if (!(entry.getKey() instanceof OlapTable)) {
671+
continue;
672+
}
673+
OlapTable pctTable = (OlapTable) entry.getKey();
674+
Set<Long> partitionIds = resetPartitionIds.computeIfAbsent(
675+
new BaseTableInfo(pctTable), key -> Sets.newHashSet());
676+
for (String pctPartitionName : entry.getValue()) {
677+
partitionIds.add(pctTable.getPartitionOrAnalysisException(pctPartitionName).getId());
678+
}
679+
}
662680
}
681+
return resetPartitionIds;
663682
}
664683

665-
private void executeWithRetry(Set<String> execPartitionNames, Map<TableIf, String> tableWithPartKey)
684+
private void executeWithRetry(Set<String> execPartitionNames, Map<TableIf, String> tableWithPartKey,
685+
Optional<IvmRewriteContext> rewriteContext)
666686
throws Exception {
667687
int retryCount = 0;
668688
int retryTime = Config.max_query_retry_time;
669689
retryTime = retryTime <= 0 ? 1 : retryTime + 1;
670690
Exception lastException = null;
671691
while (retryCount < retryTime) {
672692
try {
673-
exec(execPartitionNames, tableWithPartKey);
693+
exec(execPartitionNames, tableWithPartKey, rewriteContext);
674694
break; // Exit loop if execution is successful
675695
} catch (Exception e) {
676696
if (!(Config.isCloudMode() && SystemInfoService.needRetryWithReplan(e.getMessage()))) {
@@ -699,8 +719,8 @@ private void executeWithRetry(Set<String> execPartitionNames, Map<TableIf, Strin
699719
}
700720
}
701721

702-
private void exec(Set<String> refreshPartitionNames,
703-
Map<TableIf, String> tableWithPartKey)
722+
private void exec(Set<String> refreshPartitionNames, Map<TableIf, String> tableWithPartKey,
723+
Optional<IvmRewriteContext> rewriteContext)
704724
throws Exception {
705725
// Create MTMV context first so that new StatementContext() captures the
706726
// correct thread-local ConnectContext (with MTMV disabled rules, etc.).
@@ -715,9 +735,7 @@ private void exec(Set<String> refreshPartitionNames,
715735
mtmvCtx.setStatementContext(statementContext);
716736
statementContext.setConnectContext(mtmvCtx);
717737
statementContext.setExcludedTriggerTables(mtmv.getExcludedTriggerTables());
718-
if (mtmv.isIvm()) {
719-
statementContext.setIvmRewriteContext(Optional.of(IvmRewriteContext.full(mtmv)));
720-
}
738+
statementContext.setIvmRewriteContext(rewriteContext);
721739
for (Entry<MvccTableInfo, MvccSnapshot> entry : snapshots.entrySet()) {
722740
statementContext.setSnapshot(entry.getKey(), entry.getValue());
723741
}
@@ -1001,7 +1019,7 @@ private Map<TableIf, String> getIncrementalTableMap() throws AnalysisException {
10011019
private MTMVTaskRefreshMode generateRefreshMode(List<String> needRefreshPartitionIds) {
10021020
if (CollectionUtils.isEmpty(needRefreshPartitionIds)) {
10031021
return MTMVTaskRefreshMode.NOT_REFRESH;
1004-
} else if (needRefreshPartitionIds.size() == mtmv.getPartitionNames().size()) {
1022+
} else if (Sets.newHashSet(needRefreshPartitionIds).equals(mtmv.getPartitionNames())) {
10051023
return MTMVTaskRefreshMode.COMPLETE;
10061024
} else {
10071025
return MTMVTaskRefreshMode.PARTIAL;

‎fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmInfo.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,9 @@ public class IvmInfo {
4040
@SerializedName("rv")
4141
private long refreshVersion;
4242

43+
// In-process token used to detect metadata changes during a multi-transaction COMPLETE refresh.
44+
private transient long ivmBinlogBrokenGeneration;
45+
4346
public IvmInfo() {
4447
}
4548

@@ -48,6 +51,7 @@ public IvmInfo(IvmInfo other) {
4851
this.binlogBroken = other.binlogBroken;
4952
this.planSignature = other.planSignature;
5053
this.refreshVersion = other.refreshVersion;
54+
this.ivmBinlogBrokenGeneration = other.ivmBinlogBrokenGeneration;
5155
}
5256

5357
public boolean isEnableIvm() {
@@ -82,6 +86,14 @@ public void advanceRefreshVersion() {
8286
refreshVersion++;
8387
}
8488

89+
public void increaseBinlogBrokenGeneration() {
90+
ivmBinlogBrokenGeneration++;
91+
}
92+
93+
public long getBinlogBrokenGeneration() {
94+
return ivmBinlogBrokenGeneration;
95+
}
96+
8597
@Override
8698
public String toString() {
8799
return "IvmInfo{"

‎fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmRefreshContext.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@
2929
import java.util.Objects;
3030

3131
/**
32-
* Shared immutable context for one FE-side incremental refresh attempt.
32+
* Shared context for one FE-side incremental refresh attempt.
3333
*
3434
* <p>TSO positions are now obtained from {@code OlapTableStream} per-partition offsets
3535
* in {@link IvmDeltaRewriteState} rather than stored here.

0 commit comments

Comments
 (0)