Skip to content

Commit 9a2d20b

Browse files
seawindeyujun777
authored andcommitted
[fix](fe) Preserve IVM fallback stream offsets (apache#23)
IVM `PARTITIONS` and `COMPLETE` fallback refreshes rebuilt MV data without binding the full-image read boundary to the stream offset committed by the insert transaction. A concurrent base-table write could therefore make the rebuilt image and committed offset diverge. Multi-batch refreshes could also publish a non-PCT offset before every affected MV partition was rebuilt, while an old `COMPLETE` task could clear a newer binlog-broken event. Root cause: `MTMVTask` captured refresh versions separately from live stream updates, and RESET/SNAPSHOT planning did not carry one immutable `prev/endTso` boundary through scan planning and transaction commit. ```mermaid flowchart LR A[MTMVTask capture] -->|versions, TSO, prev/next| B[StatementContext] B --> C[RESET or SNAPSHOT scan] C --> D[MV insert transaction] D -->|RESET only| E[Commit stream next] A --> F[COMPLETE broken generation] F --> G[Publish baseline after all batches] ``` Key changes: | Area | Change | | --- | --- | | Refresh orchestration | Capture PCT boundaries per batch and non-PCT boundaries once per task; promote PARTIAL to COMPLETE before writing when non-PCT lag exists. | | Scan planning | Rewrite only captured fallback base scans to RESET/SNAPSHOT and rebuild both modes from the same captured image. | | Offset commit | Commit PCT offsets per covered batch and non-PCT offsets only in the final COMPLETE batch. | | Baseline lifecycle | Keep COMPLETE baselines broken until all batches succeed and reject stale finalization with an in-process generation. | | Tests | Add UNIQUE MOW and DUPLICATE multi-batch fallback coverage; align the latest base branch alias test with the current `IvmRewriteResult` API. | The implementation reuses existing stream scans and insert transaction updates. It adds no SQL syntax, configuration, persistent metadata schema, or second offset lifecycle. Known excluded-table and multi-base-table delta planner cases remain separate TODOs.
1 parent 0092008 commit 9a2d20b

22 files changed

Lines changed: 1538 additions & 223 deletions

‎fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -513,6 +513,7 @@ public boolean markIvmBinlogBroken() {
513513
if (ivmInfo == null) {
514514
ivmInfo = new IvmInfo();
515515
}
516+
ivmInfo.increaseBinlogBrokenGeneration();
516517
if (ivmInfo.isBinlogBroken()) {
517518
return false;
518519
}
@@ -523,6 +524,15 @@ public boolean markIvmBinlogBroken() {
523524
}
524525
}
525526

527+
public long getIvmBinlogBrokenGeneration() {
528+
readMvLock();
529+
try {
530+
return ivmInfo == null ? 0 : ivmInfo.getBinlogBrokenGeneration();
531+
} finally {
532+
readMvUnlock();
533+
}
534+
}
535+
526536
public List<String> getInsertedColumnNames() {
527537
List<Column> columns = getBaseSchema(true);
528538
List<String> columnNames = Lists.newArrayListWithExpectedSize(columns.size());

‎fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java‎

Lines changed: 72 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,11 @@
2222
import org.apache.doris.catalog.DatabaseIf;
2323
import org.apache.doris.catalog.Env;
2424
import org.apache.doris.catalog.MTMV;
25+
import org.apache.doris.catalog.OlapTable;
2526
import org.apache.doris.catalog.ScalarType;
2627
import org.apache.doris.catalog.TableIf;
2728
import org.apache.doris.catalog.info.TableNameInfo;
29+
import org.apache.doris.catalog.stream.StreamReadMode;
2830
import org.apache.doris.cloud.qe.ComputeGroupException;
2931
import org.apache.doris.common.AnalysisException;
3032
import org.apache.doris.common.Config;
@@ -48,7 +50,6 @@
4850
import org.apache.doris.metric.MetricRepo;
4951
import org.apache.doris.mtmv.BaseColInfo;
5052
import org.apache.doris.mtmv.BaseTableInfo;
51-
import org.apache.doris.mtmv.MTMVAnalyzeQueryInfo;
5253
import org.apache.doris.mtmv.MTMVBaseTableIf;
5354
import org.apache.doris.mtmv.MTMVPartitionInfo.MTMVPartitionType;
5455
import org.apache.doris.mtmv.MTMVPartitionUtil;
@@ -238,11 +239,8 @@ private PartitionPlanningException(String message, Throwable cause) {
238239
private String ivmFallbackReason;
239240
@SerializedName("cg")
240241
private String computeGroup;
241-
// Temporarily keeps the compact current layout signature hash from the failed IVM probe.
242-
// After the fallback full refresh succeeds, this hash becomes the next incremental baseline.
243-
private String ivmFallbackPlanSignature;
244-
// Runtime-only diagnostic layout string for logging the next baseline after fallback full refresh.
245-
private String ivmFallbackPlanCanonicalString;
242+
// Runtime-only layout signature from the incremental plan that triggered fallback.
243+
private IvmPlanSignature ivmFallbackPlanSignature;
246244

247245
private MTMV mtmv;
248246
private MTMVRelation relation;
@@ -506,13 +504,24 @@ private AttemptResultType executeIvmAttempt(MTMVRefreshContext refreshContext,
506504
throw new JobException("Cannot use " + request.refreshMode
507505
+ " refresh on a materialized view without INCREMENTAL capability.");
508506
}
507+
// A strict INCREMENTAL request must reach IVM so the stream can establish its initial baseline.
508+
// Only fallback-enabled requests may rebuild a missing or invalidated snapshot with COMPLETE.
509+
if (!mtmv.hasRefreshSnapshot() && request.allowFallback) {
510+
ivmFallbackReason = "INCOMPLETE_REFRESH_SNAPSHOT";
511+
LOG.warn("IVM refresh fell back for mv={}, reason=INCOMPLETE_REFRESH_SNAPSHOT, taskId={}. "
512+
+ "Continuing with COMPLETE refresh.", mtmv.getName(), getTaskId());
513+
return AttemptResultType.FALLBACK_TO_COMPLETE;
514+
}
509515
this.completedPartitions = Lists.newCopyOnWriteArrayList();
510516
this.partitionSnapshots = Maps.newConcurrentMap();
511517
ivmFallbackPlanSignature = null;
512-
ivmFallbackPlanCanonicalString = null;
513518
// Determine which partitions need refresh, same as partition-based flow.
514519
this.needRefreshPartitions = MTMVPartitionUtil.getMTMVNeedRefreshPartitions(refreshContext,
515520
relation.getBaseTablesOneLevelAndFromView());
521+
if (mtmv.getIvmInfo().isBinlogBroken()) {
522+
return handleIvmFallbackResult(IvmRefreshResult.fallback(
523+
IvmFailureReason.BINLOG_BROKEN, "Stream binlog is marked as broken"), request);
524+
}
516525
if (CollectionUtils.isEmpty(needRefreshPartitions)) {
517526
LOG.info("IVM incremental refresh skipped for mv={}: all partitions are synced, taskId={}",
518527
mtmv.getName(), getTaskId());
@@ -552,15 +561,27 @@ private AttemptResultType executeIvmAttempt(MTMVRefreshContext refreshContext,
552561
mtmv.getName(), getTaskId());
553562
return AttemptResultType.SUCCESS;
554563
}
564+
return handleIvmFallbackResult(ivmResult, request);
565+
}
566+
567+
private AttemptResultType handleIvmFallbackResult(IvmRefreshResult ivmResult, RefreshRequest request)
568+
throws JobException {
555569
ivmFallbackReason = ivmResult.getFailureReason().name();
570+
ivmFallbackPlanSignature = ivmResult.getPlanSignature();
556571
if (!request.allowFallback) {
557572
throw new JobException(
558573
"IVM incremental refresh failed for mv=" + mtmv.getName()
559574
+ ", reason=" + ivmResult.getFailureReason()
560575
+ ", detail=" + ivmResult.getDetailMessage());
561576
}
562-
// TODO(IVM): More pre-execution failures may require direct COMPLETE
563-
// recovery, such as signature mismatch or invalid binlog state.
577+
if (ivmResult.getFailureReason() == IvmFailureReason.BINLOG_BROKEN) {
578+
// The previous task already entered the IVM execution phase. If
579+
// fallback is allowed, jump directly to COMPLETE recovery instead of
580+
// trying PARTITIONS first.
581+
LOG.warn("IVM previous run incomplete for mv={}, taskId={}. Continuing with COMPLETE recovery.",
582+
mtmv.getName(), getTaskId());
583+
return AttemptResultType.FALLBACK_TO_COMPLETE;
584+
}
564585
if (ivmResult.getFailureReason() == IvmFailureReason.PLAN_SIGNATURE_MISMATCH) {
565586
LOG.warn("IVM refresh fell back for mv={}, reason={}, detail={}, taskId={}. "
566587
+ "Continuing with COMPLETE refresh.",
@@ -597,29 +618,36 @@ private boolean executePartitionBasedRefresh(MTMVRefreshContext refreshContext,
597618

598619
private void executePartitionBasedRefresh(MTMVRefreshContext context)
599620
throws JobException, AnalysisException {
621+
boolean useIvmFallbackStreams = mtmv.isIvm();
600622
Map<TableIf, String> tableWithPartKey = getIncrementalTableMap();
623+
long baselineGeneration = useIvmFallbackStreams
624+
? IvmRefreshManager.markIvmBaselineBroken(mtmv) : -1;
601625
this.completedPartitions = Lists.newCopyOnWriteArrayList();
602626
int refreshPartitionNum = mtmv.getRefreshPartitionNum();
603627
long execNum = (needRefreshPartitions.size() / refreshPartitionNum) + ((needRefreshPartitions.size()
604628
% refreshPartitionNum) > 0 ? 1 : 0);
629+
boolean refreshAllPartitions = Sets.newHashSet(needRefreshPartitions).equals(mtmv.getPartitionNames());
605630
this.partitionSnapshots = Maps.newConcurrentMap();
606631
for (int i = 0; i < execNum; i++) {
607632
int start = i * refreshPartitionNum;
608633
int end = start + refreshPartitionNum;
609634
Set<String> execPartitionNames = Sets.newHashSet(needRefreshPartitions
610635
.subList(start, Math.min(end, needRefreshPartitions.size())));
636+
Map<BaseTableInfo, Set<Long>> batchResetPartitionIds = useIvmFallbackStreams
637+
? collectPctResetPartitionIds(context, execPartitionNames) : Maps.newHashMap();
638+
Optional<IvmRewriteContext> rewriteContext = Optional.empty();
639+
if (useIvmFallbackStreams) {
640+
StreamReadMode nonPctReadMode = i == 0 && refreshAllPartitions
641+
? StreamReadMode.RESET : StreamReadMode.SNAPSHOT;
642+
rewriteContext = Optional.of(
643+
IvmRewriteContext.full(mtmv, batchResetPartitionIds, nonPctReadMode));
644+
}
611645
// need get names before exec
612646
Map<String, MTMVRefreshPartitionSnapshot> execPartitionSnapshots = MTMVPartitionUtil
613647
.generatePartitionSnapshots(context, relation.getBaseTablesOneLevelAndFromView(),
614648
execPartitionNames);
615649
try {
616-
// TODO(IVM): When IVM full refresh falls back here, the refresh SQL should
617-
// bind to a specific TSO snapshot to guarantee that consumedTso exactly matches
618-
// the version the SQL actually reads. Currently TSO support is incomplete, so
619-
// the full refresh reads the latest visible version without TSO binding, so
620-
// ivmPreRefreshTsos capture/reset stays disabled until TSO-bound reads are
621-
// implemented.
622-
executeWithRetry(execPartitionNames, tableWithPartKey);
650+
executeWithRetry(execPartitionNames, tableWithPartKey, rewriteContext);
623651
} catch (Exception e) {
624652
LOG.error("Execution failed after retries, mvName: {}, taskId: {}",
625653
mtmv.getName(), getTaskId(), e);
@@ -628,53 +656,45 @@ private void executePartitionBasedRefresh(MTMVRefreshContext context)
628656
completedPartitions.addAll(execPartitionNames);
629657
partitionSnapshots.putAll(execPartitionSnapshots);
630658
}
631-
if (mtmv.isIvm()) {
632-
updateIvmPlanSignatureAfterFullRefreshIfNeeded();
659+
if (useIvmFallbackStreams) {
660+
IvmRefreshManager.finishIvmFullRefresh(mtmv, baselineGeneration,
661+
ivmFallbackPlanSignature);
662+
ivmFallbackPlanSignature = null;
633663
}
634664
LOG.info("MTMVTask refresh used snapshot: {}, mvDbName: {}, mvName: {}, taskId: {}", partitionSnapshots,
635665
mtmv.getDatabase().getFullName(), mtmv.getName(), getTaskId());
636666
}
637667

638-
private void updateIvmPlanSignatureAfterFullRefreshIfNeeded() throws JobException {
639-
if (IvmFailureReason.PLAN_SIGNATURE_MISMATCH.name().equals(ivmFallbackReason)) {
640-
refreshIvmPlanSignatureAfterFullRefresh();
641-
return;
642-
}
643-
if (ivmFallbackPlanSignature == null) {
644-
return;
645-
}
646-
IvmRefreshManager.updatePlanSignatureAfterFullRefresh(
647-
mtmv, ivmFallbackPlanSignature, ivmFallbackPlanCanonicalString);
648-
ivmFallbackPlanSignature = null;
649-
ivmFallbackPlanCanonicalString = null;
650-
}
651-
652-
private void refreshIvmPlanSignatureAfterFullRefresh() throws JobException {
653-
try {
654-
ConnectContext ctx = MTMVPlanUtil.createMTMVContext(
655-
mtmv, MTMVPlanUtil.DISABLE_RULES_WHEN_RUN_MTMV_TASK);
656-
MTMVAnalyzeQueryInfo analyzeQueryInfo = MTMVPlanUtil.analyzeQueryWithSql(mtmv, ctx,
657-
Optional.of(IvmRewriteContext.normalize(mtmv)));
658-
IvmPlanSignature currentSignature = analyzeQueryInfo.getIvmRewriteResult().getPlanSignature();
659-
IvmRefreshManager.updatePlanSignatureAfterFullRefresh(
660-
mtmv, currentSignature.getSha256(), currentSignature.getCanonicalString());
661-
ivmFallbackPlanSignature = null;
662-
ivmFallbackPlanCanonicalString = null;
663-
} catch (Exception e) {
664-
throw new JobException("Failed to rebuild IVM plan signature after full refresh, mv="
665-
+ mtmv.getName() + ", detail=" + e.getMessage(), e);
668+
private Map<BaseTableInfo, Set<Long>> collectPctResetPartitionIds(MTMVRefreshContext context,
669+
Set<String> execPartitionNames) throws AnalysisException {
670+
Map<BaseTableInfo, Set<Long>> resetPartitionIds = Maps.newHashMap();
671+
for (String mvPartitionName : execPartitionNames) {
672+
for (Entry<MTMVRelatedTableIf, Set<String>> entry
673+
: context.getByPartitionName(mvPartitionName).entrySet()) {
674+
if (!(entry.getKey() instanceof OlapTable)) {
675+
continue;
676+
}
677+
OlapTable pctTable = (OlapTable) entry.getKey();
678+
Set<Long> partitionIds = resetPartitionIds.computeIfAbsent(
679+
new BaseTableInfo(pctTable), key -> Sets.newHashSet());
680+
for (String pctPartitionName : entry.getValue()) {
681+
partitionIds.add(pctTable.getPartitionOrAnalysisException(pctPartitionName).getId());
682+
}
683+
}
666684
}
685+
return resetPartitionIds;
667686
}
668687

669-
private void executeWithRetry(Set<String> execPartitionNames, Map<TableIf, String> tableWithPartKey)
688+
private void executeWithRetry(Set<String> execPartitionNames, Map<TableIf, String> tableWithPartKey,
689+
Optional<IvmRewriteContext> rewriteContext)
670690
throws Exception {
671691
int retryCount = 0;
672692
int retryTime = Config.max_query_retry_time;
673693
retryTime = retryTime <= 0 ? 1 : retryTime + 1;
674694
Exception lastException = null;
675695
while (retryCount < retryTime) {
676696
try {
677-
exec(execPartitionNames, tableWithPartKey);
697+
exec(execPartitionNames, tableWithPartKey, rewriteContext);
678698
break; // Exit loop if execution is successful
679699
} catch (Exception e) {
680700
if (!(Config.isCloudMode() && SystemInfoService.needRetryWithReplan(e.getMessage()))) {
@@ -703,8 +723,8 @@ private void executeWithRetry(Set<String> execPartitionNames, Map<TableIf, Strin
703723
}
704724
}
705725

706-
private void exec(Set<String> refreshPartitionNames,
707-
Map<TableIf, String> tableWithPartKey)
726+
private void exec(Set<String> refreshPartitionNames, Map<TableIf, String> tableWithPartKey,
727+
Optional<IvmRewriteContext> rewriteContext)
708728
throws Exception {
709729
// Create MTMV context first so that new StatementContext() captures the
710730
// correct thread-local ConnectContext (with MTMV disabled rules, etc.).
@@ -719,9 +739,7 @@ private void exec(Set<String> refreshPartitionNames,
719739
mtmvCtx.setStatementContext(statementContext);
720740
statementContext.setConnectContext(mtmvCtx);
721741
statementContext.setExcludedTriggerTables(mtmv.getExcludedTriggerTables());
722-
if (mtmv.isIvm()) {
723-
statementContext.setIvmRewriteContext(Optional.of(IvmRewriteContext.full(mtmv)));
724-
}
742+
statementContext.setIvmRewriteContext(rewriteContext);
725743
for (Entry<MvccTableInfo, MvccSnapshot> entry : snapshots.entrySet()) {
726744
statementContext.setSnapshot(entry.getKey(), entry.getValue());
727745
}
@@ -1005,7 +1023,7 @@ private Map<TableIf, String> getIncrementalTableMap() throws AnalysisException {
10051023
private MTMVTaskRefreshMode generateRefreshMode(List<String> needRefreshPartitionIds) {
10061024
if (CollectionUtils.isEmpty(needRefreshPartitionIds)) {
10071025
return MTMVTaskRefreshMode.NOT_REFRESH;
1008-
} else if (needRefreshPartitionIds.size() == mtmv.getPartitionNames().size()) {
1026+
} else if (Sets.newHashSet(needRefreshPartitionIds).equals(mtmv.getPartitionNames())) {
10091027
return MTMVTaskRefreshMode.COMPLETE;
10101028
} else {
10111029
return MTMVTaskRefreshMode.PARTIAL;

‎fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmInfo.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,9 @@ public class IvmInfo {
4040
@SerializedName("rv")
4141
private long refreshVersion;
4242

43+
// In-process token used to detect metadata changes during a multi-transaction COMPLETE refresh.
44+
private transient long ivmBinlogBrokenGeneration;
45+
4346
public IvmInfo() {
4447
}
4548

@@ -48,6 +51,7 @@ public IvmInfo(IvmInfo other) {
4851
this.binlogBroken = other.binlogBroken;
4952
this.planSignature = other.planSignature;
5053
this.refreshVersion = other.refreshVersion;
54+
this.ivmBinlogBrokenGeneration = other.ivmBinlogBrokenGeneration;
5155
}
5256

5357
public boolean isEnableIvm() {
@@ -82,6 +86,14 @@ public void advanceRefreshVersion() {
8286
refreshVersion++;
8387
}
8488

89+
public void increaseBinlogBrokenGeneration() {
90+
ivmBinlogBrokenGeneration++;
91+
}
92+
93+
public long getBinlogBrokenGeneration() {
94+
return ivmBinlogBrokenGeneration;
95+
}
96+
8597
@Override
8698
public String toString() {
8799
return "IvmInfo{"

‎fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmRefreshContext.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@
2929
import java.util.Objects;
3030

3131
/**
32-
* Shared immutable context for one FE-side incremental refresh attempt.
32+
* Shared context for one FE-side incremental refresh attempt.
3333
*
3434
* <p>TSO positions are now obtained from {@code OlapTableStream} per-partition offsets
3535
* in {@link IvmDeltaRewriteState} rather than stored here.

0 commit comments

Comments
 (0)