2222import org .apache .doris .catalog .DatabaseIf ;
2323import org .apache .doris .catalog .Env ;
2424import org .apache .doris .catalog .MTMV ;
25+ import org .apache .doris .catalog .OlapTable ;
2526import org .apache .doris .catalog .ScalarType ;
2627import org .apache .doris .catalog .TableIf ;
2728import org .apache .doris .catalog .info .TableNameInfo ;
29+ import org .apache .doris .catalog .stream .StreamReadMode ;
2830import org .apache .doris .cloud .qe .ComputeGroupException ;
2931import org .apache .doris .common .AnalysisException ;
3032import org .apache .doris .common .Config ;
4850import org .apache .doris .metric .MetricRepo ;
4951import org .apache .doris .mtmv .BaseColInfo ;
5052import org .apache .doris .mtmv .BaseTableInfo ;
51- import org .apache .doris .mtmv .MTMVAnalyzeQueryInfo ;
5253import org .apache .doris .mtmv .MTMVBaseTableIf ;
5354import org .apache .doris .mtmv .MTMVPartitionInfo .MTMVPartitionType ;
5455import org .apache .doris .mtmv .MTMVPartitionUtil ;
@@ -236,11 +237,8 @@ private PartitionPlanningException(String message, Throwable cause) {
236237 private String ivmFallbackReason ;
237238 @ SerializedName ("cg" )
238239 private String computeGroup ;
239- // Temporarily keeps the compact current layout signature hash from the failed IVM probe.
240- // After the fallback full refresh succeeds, this hash becomes the next incremental baseline.
241- private String ivmFallbackPlanSignature ;
242- // Runtime-only diagnostic layout string for logging the next baseline after fallback full refresh.
243- private String ivmFallbackPlanCanonicalString ;
240+ // Runtime-only layout signature from the incremental plan that triggered fallback.
241+ private IvmPlanSignature ivmFallbackPlanSignature ;
244242
245243 private MTMV mtmv ;
246244 private MTMVRelation relation ;
@@ -503,13 +501,24 @@ private AttemptResultType executeIvmAttempt(MTMVRefreshContext refreshContext,
503501 throw new JobException ("Cannot use " + request .refreshMode
504502 + " refresh on a materialized view without INCREMENTAL capability." );
505503 }
504+ // A strict INCREMENTAL request must reach IVM so the stream can establish its initial baseline.
505+ // Only fallback-enabled requests may rebuild a missing or invalidated snapshot with COMPLETE.
506+ if (!mtmv .hasRefreshSnapshot () && request .allowFallback ) {
507+ ivmFallbackReason = "INCOMPLETE_REFRESH_SNAPSHOT" ;
508+ LOG .warn ("IVM refresh fell back for mv={}, reason=INCOMPLETE_REFRESH_SNAPSHOT, taskId={}. "
509+ + "Continuing with COMPLETE refresh." , mtmv .getName (), getTaskId ());
510+ return AttemptResultType .FALLBACK_TO_COMPLETE ;
511+ }
506512 this .completedPartitions = Lists .newCopyOnWriteArrayList ();
507513 this .partitionSnapshots = Maps .newConcurrentMap ();
508514 ivmFallbackPlanSignature = null ;
509- ivmFallbackPlanCanonicalString = null ;
510515 // Determine which partitions need refresh, same as partition-based flow.
511516 this .needRefreshPartitions = MTMVPartitionUtil .getMTMVNeedRefreshPartitions (refreshContext ,
512517 relation .getBaseTablesOneLevelAndFromView ());
518+ if (mtmv .getIvmInfo ().isBinlogBroken ()) {
519+ return handleIvmFallbackResult (IvmRefreshResult .fallback (
520+ IvmFailureReason .BINLOG_BROKEN , "Stream binlog is marked as broken" ), request );
521+ }
513522 if (CollectionUtils .isEmpty (needRefreshPartitions )) {
514523 LOG .info ("IVM incremental refresh skipped for mv={}: all partitions are synced, taskId={}" ,
515524 mtmv .getName (), getTaskId ());
@@ -549,15 +558,27 @@ private AttemptResultType executeIvmAttempt(MTMVRefreshContext refreshContext,
549558 mtmv .getName (), getTaskId ());
550559 return AttemptResultType .SUCCESS ;
551560 }
561+ return handleIvmFallbackResult (ivmResult , request );
562+ }
563+
564+ private AttemptResultType handleIvmFallbackResult (IvmRefreshResult ivmResult , RefreshRequest request )
565+ throws JobException {
552566 ivmFallbackReason = ivmResult .getFailureReason ().name ();
567+ ivmFallbackPlanSignature = ivmResult .getPlanSignature ();
553568 if (!request .allowFallback ) {
554569 throw new JobException (
555570 "IVM incremental refresh failed for mv=" + mtmv .getName ()
556571 + ", reason=" + ivmResult .getFailureReason ()
557572 + ", detail=" + ivmResult .getDetailMessage ());
558573 }
559- // TODO(IVM): More pre-execution failures may require direct COMPLETE
560- // recovery, such as signature mismatch or invalid binlog state.
574+ if (ivmResult .getFailureReason () == IvmFailureReason .BINLOG_BROKEN ) {
575+ // The previous task already entered the IVM execution phase. If
576+ // fallback is allowed, jump directly to COMPLETE recovery instead of
577+ // trying PARTITIONS first.
578+ LOG .warn ("IVM previous run incomplete for mv={}, taskId={}. Continuing with COMPLETE recovery." ,
579+ mtmv .getName (), getTaskId ());
580+ return AttemptResultType .FALLBACK_TO_COMPLETE ;
581+ }
561582 if (ivmResult .getFailureReason () == IvmFailureReason .PLAN_SIGNATURE_MISMATCH ) {
562583 LOG .warn ("IVM refresh fell back for mv={}, reason={}, detail={}, taskId={}. "
563584 + "Continuing with COMPLETE refresh." ,
@@ -594,83 +615,82 @@ private boolean executePartitionBasedRefresh(MTMVRefreshContext refreshContext,
594615
595616 private void executePartitionBasedRefresh (MTMVRefreshContext context )
596617 throws JobException , AnalysisException {
618+ boolean useIvmFallbackStreams = mtmv .isIvm ();
597619 Map <TableIf , String > tableWithPartKey = getIncrementalTableMap ();
620+ long baselineGeneration = useIvmFallbackStreams
621+ ? IvmRefreshManager .markIvmBaselineBroken (mtmv ) : -1 ;
598622 this .completedPartitions = Lists .newCopyOnWriteArrayList ();
599623 int refreshPartitionNum = mtmv .getRefreshPartitionNum ();
600624 long execNum = (needRefreshPartitions .size () / refreshPartitionNum ) + ((needRefreshPartitions .size ()
601625 % refreshPartitionNum ) > 0 ? 1 : 0 );
626+ boolean refreshAllPartitions = Sets .newHashSet (needRefreshPartitions ).equals (mtmv .getPartitionNames ());
602627 this .partitionSnapshots = Maps .newConcurrentMap ();
603628 for (int i = 0 ; i < execNum ; i ++) {
604629 int start = i * refreshPartitionNum ;
605630 int end = start + refreshPartitionNum ;
606631 Set <String > execPartitionNames = Sets .newHashSet (needRefreshPartitions
607632 .subList (start , Math .min (end , needRefreshPartitions .size ())));
633+ Map <BaseTableInfo , Set <Long >> batchResetPartitionIds = useIvmFallbackStreams
634+ ? collectPctResetPartitionIds (context , execPartitionNames ) : Maps .newHashMap ();
635+ Optional <IvmRewriteContext > rewriteContext = Optional .empty ();
636+ if (useIvmFallbackStreams ) {
637+ StreamReadMode nonPctReadMode = i == 0 && refreshAllPartitions
638+ ? StreamReadMode .RESET : StreamReadMode .SNAPSHOT ;
639+ rewriteContext = Optional .of (
640+ IvmRewriteContext .full (mtmv , batchResetPartitionIds , nonPctReadMode ));
641+ }
608642 // need get names before exec
609643 Map <String , MTMVRefreshPartitionSnapshot > execPartitionSnapshots = MTMVPartitionUtil
610644 .generatePartitionSnapshots (context , relation .getBaseTablesOneLevelAndFromView (),
611645 execPartitionNames );
612646 try {
613- // TODO(IVM): When IVM full refresh falls back here, the refresh SQL should
614- // bind to a specific TSO snapshot to guarantee that consumedTso exactly matches
615- // the version the SQL actually reads. Currently TSO support is incomplete, so
616- // the full refresh reads the latest visible version without TSO binding, so
617- // ivmPreRefreshTsos capture/reset stays disabled until TSO-bound reads are
618- // implemented.
619- executeWithRetry (execPartitionNames , tableWithPartKey );
647+ executeWithRetry (execPartitionNames , tableWithPartKey , rewriteContext );
620648 } catch (Exception e ) {
621649 LOG .error ("Execution failed after retries: {}" , e .getMessage ());
622650 throw new JobException (e .getMessage (), e );
623651 }
624652 completedPartitions .addAll (execPartitionNames );
625653 partitionSnapshots .putAll (execPartitionSnapshots );
626654 }
627- if (mtmv .isIvm ()) {
628- updateIvmPlanSignatureAfterFullRefreshIfNeeded ();
655+ if (useIvmFallbackStreams ) {
656+ IvmRefreshManager .finishIvmFullRefresh (mtmv , baselineGeneration ,
657+ ivmFallbackPlanSignature );
658+ ivmFallbackPlanSignature = null ;
629659 }
630660 LOG .info ("MTMVTask refresh used snapshot: {}, mvDbName: {}, mvName: {}, taskId: {}" , partitionSnapshots ,
631661 mtmv .getDatabase ().getFullName (), mtmv .getName (), getTaskId ());
632662 }
633663
634- private void updateIvmPlanSignatureAfterFullRefreshIfNeeded () throws JobException {
635- if (IvmFailureReason .PLAN_SIGNATURE_MISMATCH .name ().equals (ivmFallbackReason )) {
636- refreshIvmPlanSignatureAfterFullRefresh ();
637- return ;
638- }
639- if (ivmFallbackPlanSignature == null ) {
640- return ;
641- }
642- IvmRefreshManager .updatePlanSignatureAfterFullRefresh (
643- mtmv , ivmFallbackPlanSignature , ivmFallbackPlanCanonicalString );
644- ivmFallbackPlanSignature = null ;
645- ivmFallbackPlanCanonicalString = null ;
646- }
647-
648- private void refreshIvmPlanSignatureAfterFullRefresh () throws JobException {
649- try {
650- ConnectContext ctx = MTMVPlanUtil .createMTMVContext (
651- mtmv , MTMVPlanUtil .DISABLE_RULES_WHEN_RUN_MTMV_TASK );
652- MTMVAnalyzeQueryInfo analyzeQueryInfo = MTMVPlanUtil .analyzeQueryWithSql (mtmv , ctx ,
653- Optional .of (IvmRewriteContext .normalize (mtmv )));
654- IvmPlanSignature currentSignature = analyzeQueryInfo .getIvmRewriteResult ().getPlanSignature ();
655- IvmRefreshManager .updatePlanSignatureAfterFullRefresh (
656- mtmv , currentSignature .getSha256 (), currentSignature .getCanonicalString ());
657- ivmFallbackPlanSignature = null ;
658- ivmFallbackPlanCanonicalString = null ;
659- } catch (Exception e ) {
660- throw new JobException ("Failed to rebuild IVM plan signature after full refresh, mv="
661- + mtmv .getName () + ", detail=" + e .getMessage (), e );
664+ private Map <BaseTableInfo , Set <Long >> collectPctResetPartitionIds (MTMVRefreshContext context ,
665+ Set <String > execPartitionNames ) throws AnalysisException {
666+ Map <BaseTableInfo , Set <Long >> resetPartitionIds = Maps .newHashMap ();
667+ for (String mvPartitionName : execPartitionNames ) {
668+ for (Entry <MTMVRelatedTableIf , Set <String >> entry
669+ : context .getByPartitionName (mvPartitionName ).entrySet ()) {
670+ if (!(entry .getKey () instanceof OlapTable )) {
671+ continue ;
672+ }
673+ OlapTable pctTable = (OlapTable ) entry .getKey ();
674+ Set <Long > partitionIds = resetPartitionIds .computeIfAbsent (
675+ new BaseTableInfo (pctTable ), key -> Sets .newHashSet ());
676+ for (String pctPartitionName : entry .getValue ()) {
677+ partitionIds .add (pctTable .getPartitionOrAnalysisException (pctPartitionName ).getId ());
678+ }
679+ }
662680 }
681+ return resetPartitionIds ;
663682 }
664683
665- private void executeWithRetry (Set <String > execPartitionNames , Map <TableIf , String > tableWithPartKey )
684+ private void executeWithRetry (Set <String > execPartitionNames , Map <TableIf , String > tableWithPartKey ,
685+ Optional <IvmRewriteContext > rewriteContext )
666686 throws Exception {
667687 int retryCount = 0 ;
668688 int retryTime = Config .max_query_retry_time ;
669689 retryTime = retryTime <= 0 ? 1 : retryTime + 1 ;
670690 Exception lastException = null ;
671691 while (retryCount < retryTime ) {
672692 try {
673- exec (execPartitionNames , tableWithPartKey );
693+ exec (execPartitionNames , tableWithPartKey , rewriteContext );
674694 break ; // Exit loop if execution is successful
675695 } catch (Exception e ) {
676696 if (!(Config .isCloudMode () && SystemInfoService .needRetryWithReplan (e .getMessage ()))) {
@@ -699,8 +719,8 @@ private void executeWithRetry(Set<String> execPartitionNames, Map<TableIf, Strin
699719 }
700720 }
701721
702- private void exec (Set <String > refreshPartitionNames ,
703- Map < TableIf , String > tableWithPartKey )
722+ private void exec (Set <String > refreshPartitionNames , Map < TableIf , String > tableWithPartKey ,
723+ Optional < IvmRewriteContext > rewriteContext )
704724 throws Exception {
705725 // Create MTMV context first so that new StatementContext() captures the
706726 // correct thread-local ConnectContext (with MTMV disabled rules, etc.).
@@ -715,9 +735,7 @@ private void exec(Set<String> refreshPartitionNames,
715735 mtmvCtx .setStatementContext (statementContext );
716736 statementContext .setConnectContext (mtmvCtx );
717737 statementContext .setExcludedTriggerTables (mtmv .getExcludedTriggerTables ());
718- if (mtmv .isIvm ()) {
719- statementContext .setIvmRewriteContext (Optional .of (IvmRewriteContext .full (mtmv )));
720- }
738+ statementContext .setIvmRewriteContext (rewriteContext );
721739 for (Entry <MvccTableInfo , MvccSnapshot > entry : snapshots .entrySet ()) {
722740 statementContext .setSnapshot (entry .getKey (), entry .getValue ());
723741 }
@@ -1001,7 +1019,7 @@ private Map<TableIf, String> getIncrementalTableMap() throws AnalysisException {
10011019 private MTMVTaskRefreshMode generateRefreshMode (List <String > needRefreshPartitionIds ) {
10021020 if (CollectionUtils .isEmpty (needRefreshPartitionIds )) {
10031021 return MTMVTaskRefreshMode .NOT_REFRESH ;
1004- } else if (needRefreshPartitionIds . size () == mtmv .getPartitionNames (). size ( )) {
1022+ } else if (Sets . newHashSet ( needRefreshPartitionIds ). equals ( mtmv .getPartitionNames ())) {
10051023 return MTMVTaskRefreshMode .COMPLETE ;
10061024 } else {
10071025 return MTMVTaskRefreshMode .PARTIAL ;
0 commit comments