2222import org .apache .doris .catalog .DatabaseIf ;
2323import org .apache .doris .catalog .Env ;
2424import org .apache .doris .catalog .MTMV ;
25+ import org .apache .doris .catalog .OlapTable ;
2526import org .apache .doris .catalog .ScalarType ;
2627import org .apache .doris .catalog .TableIf ;
2728import org .apache .doris .catalog .info .TableNameInfo ;
29+ import org .apache .doris .catalog .stream .StreamReadMode ;
2830import org .apache .doris .cloud .qe .ComputeGroupException ;
2931import org .apache .doris .common .AnalysisException ;
3032import org .apache .doris .common .Config ;
4850import org .apache .doris .metric .MetricRepo ;
4951import org .apache .doris .mtmv .BaseColInfo ;
5052import org .apache .doris .mtmv .BaseTableInfo ;
51- import org .apache .doris .mtmv .MTMVAnalyzeQueryInfo ;
5253import org .apache .doris .mtmv .MTMVBaseTableIf ;
5354import org .apache .doris .mtmv .MTMVPartitionInfo .MTMVPartitionType ;
5455import org .apache .doris .mtmv .MTMVPartitionUtil ;
@@ -238,11 +239,8 @@ private PartitionPlanningException(String message, Throwable cause) {
238239 private String ivmFallbackReason ;
239240 @ SerializedName ("cg" )
240241 private String computeGroup ;
241- // Temporarily keeps the compact current layout signature hash from the failed IVM probe.
242- // After the fallback full refresh succeeds, this hash becomes the next incremental baseline.
243- private String ivmFallbackPlanSignature ;
244- // Runtime-only diagnostic layout string for logging the next baseline after fallback full refresh.
245- private String ivmFallbackPlanCanonicalString ;
242+ // Runtime-only layout signature from the incremental plan that triggered fallback.
243+ private IvmPlanSignature ivmFallbackPlanSignature ;
246244
247245 private MTMV mtmv ;
248246 private MTMVRelation relation ;
@@ -506,13 +504,24 @@ private AttemptResultType executeIvmAttempt(MTMVRefreshContext refreshContext,
506504 throw new JobException ("Cannot use " + request .refreshMode
507505 + " refresh on a materialized view without INCREMENTAL capability." );
508506 }
507+ // A strict INCREMENTAL request must reach IVM so the stream can establish its initial baseline.
508+ // Only fallback-enabled requests may rebuild a missing or invalidated snapshot with COMPLETE.
509+ if (!mtmv .hasRefreshSnapshot () && request .allowFallback ) {
510+ ivmFallbackReason = "INCOMPLETE_REFRESH_SNAPSHOT" ;
511+ LOG .warn ("IVM refresh fell back for mv={}, reason=INCOMPLETE_REFRESH_SNAPSHOT, taskId={}. "
512+ + "Continuing with COMPLETE refresh." , mtmv .getName (), getTaskId ());
513+ return AttemptResultType .FALLBACK_TO_COMPLETE ;
514+ }
509515 this .completedPartitions = Lists .newCopyOnWriteArrayList ();
510516 this .partitionSnapshots = Maps .newConcurrentMap ();
511517 ivmFallbackPlanSignature = null ;
512- ivmFallbackPlanCanonicalString = null ;
513518 // Determine which partitions need refresh, same as partition-based flow.
514519 this .needRefreshPartitions = MTMVPartitionUtil .getMTMVNeedRefreshPartitions (refreshContext ,
515520 relation .getBaseTablesOneLevelAndFromView ());
521+ if (mtmv .getIvmInfo ().isBinlogBroken ()) {
522+ return handleIvmFallbackResult (IvmRefreshResult .fallback (
523+ IvmFailureReason .BINLOG_BROKEN , "Stream binlog is marked as broken" ), request );
524+ }
516525 if (CollectionUtils .isEmpty (needRefreshPartitions )) {
517526 LOG .info ("IVM incremental refresh skipped for mv={}: all partitions are synced, taskId={}" ,
518527 mtmv .getName (), getTaskId ());
@@ -552,15 +561,27 @@ private AttemptResultType executeIvmAttempt(MTMVRefreshContext refreshContext,
552561 mtmv .getName (), getTaskId ());
553562 return AttemptResultType .SUCCESS ;
554563 }
564+ return handleIvmFallbackResult (ivmResult , request );
565+ }
566+
567+ private AttemptResultType handleIvmFallbackResult (IvmRefreshResult ivmResult , RefreshRequest request )
568+ throws JobException {
555569 ivmFallbackReason = ivmResult .getFailureReason ().name ();
570+ ivmFallbackPlanSignature = ivmResult .getPlanSignature ();
556571 if (!request .allowFallback ) {
557572 throw new JobException (
558573 "IVM incremental refresh failed for mv=" + mtmv .getName ()
559574 + ", reason=" + ivmResult .getFailureReason ()
560575 + ", detail=" + ivmResult .getDetailMessage ());
561576 }
562- // TODO(IVM): More pre-execution failures may require direct COMPLETE
563- // recovery, such as signature mismatch or invalid binlog state.
577+ if (ivmResult .getFailureReason () == IvmFailureReason .BINLOG_BROKEN ) {
578+ // The previous task already entered the IVM execution phase. If
579+ // fallback is allowed, jump directly to COMPLETE recovery instead of
580+ // trying PARTITIONS first.
581+ LOG .warn ("IVM previous run incomplete for mv={}, taskId={}. Continuing with COMPLETE recovery." ,
582+ mtmv .getName (), getTaskId ());
583+ return AttemptResultType .FALLBACK_TO_COMPLETE ;
584+ }
564585 if (ivmResult .getFailureReason () == IvmFailureReason .PLAN_SIGNATURE_MISMATCH ) {
565586 LOG .warn ("IVM refresh fell back for mv={}, reason={}, detail={}, taskId={}. "
566587 + "Continuing with COMPLETE refresh." ,
@@ -597,29 +618,36 @@ private boolean executePartitionBasedRefresh(MTMVRefreshContext refreshContext,
597618
598619 private void executePartitionBasedRefresh (MTMVRefreshContext context )
599620 throws JobException , AnalysisException {
621+ boolean useIvmFallbackStreams = mtmv .isIvm ();
600622 Map <TableIf , String > tableWithPartKey = getIncrementalTableMap ();
623+ long baselineGeneration = useIvmFallbackStreams
624+ ? IvmRefreshManager .markIvmBaselineBroken (mtmv ) : -1 ;
601625 this .completedPartitions = Lists .newCopyOnWriteArrayList ();
602626 int refreshPartitionNum = mtmv .getRefreshPartitionNum ();
603627 long execNum = (needRefreshPartitions .size () / refreshPartitionNum ) + ((needRefreshPartitions .size ()
604628 % refreshPartitionNum ) > 0 ? 1 : 0 );
629+ boolean refreshAllPartitions = Sets .newHashSet (needRefreshPartitions ).equals (mtmv .getPartitionNames ());
605630 this .partitionSnapshots = Maps .newConcurrentMap ();
606631 for (int i = 0 ; i < execNum ; i ++) {
607632 int start = i * refreshPartitionNum ;
608633 int end = start + refreshPartitionNum ;
609634 Set <String > execPartitionNames = Sets .newHashSet (needRefreshPartitions
610635 .subList (start , Math .min (end , needRefreshPartitions .size ())));
636+ Map <BaseTableInfo , Set <Long >> batchResetPartitionIds = useIvmFallbackStreams
637+ ? collectPctResetPartitionIds (context , execPartitionNames ) : Maps .newHashMap ();
638+ Optional <IvmRewriteContext > rewriteContext = Optional .empty ();
639+ if (useIvmFallbackStreams ) {
640+ StreamReadMode nonPctReadMode = i == 0 && refreshAllPartitions
641+ ? StreamReadMode .RESET : StreamReadMode .SNAPSHOT ;
642+ rewriteContext = Optional .of (
643+ IvmRewriteContext .full (mtmv , batchResetPartitionIds , nonPctReadMode ));
644+ }
611645 // need get names before exec
612646 Map <String , MTMVRefreshPartitionSnapshot > execPartitionSnapshots = MTMVPartitionUtil
613647 .generatePartitionSnapshots (context , relation .getBaseTablesOneLevelAndFromView (),
614648 execPartitionNames );
615649 try {
616- // TODO(IVM): When IVM full refresh falls back here, the refresh SQL should
617- // bind to a specific TSO snapshot to guarantee that consumedTso exactly matches
618- // the version the SQL actually reads. Currently TSO support is incomplete, so
619- // the full refresh reads the latest visible version without TSO binding, so
620- // ivmPreRefreshTsos capture/reset stays disabled until TSO-bound reads are
621- // implemented.
622- executeWithRetry (execPartitionNames , tableWithPartKey );
650+ executeWithRetry (execPartitionNames , tableWithPartKey , rewriteContext );
623651 } catch (Exception e ) {
624652 LOG .error ("Execution failed after retries, mvName: {}, taskId: {}" ,
625653 mtmv .getName (), getTaskId (), e );
@@ -628,53 +656,45 @@ private void executePartitionBasedRefresh(MTMVRefreshContext context)
628656 completedPartitions .addAll (execPartitionNames );
629657 partitionSnapshots .putAll (execPartitionSnapshots );
630658 }
631- if (mtmv .isIvm ()) {
632- updateIvmPlanSignatureAfterFullRefreshIfNeeded ();
659+ if (useIvmFallbackStreams ) {
660+ IvmRefreshManager .finishIvmFullRefresh (mtmv , baselineGeneration ,
661+ ivmFallbackPlanSignature );
662+ ivmFallbackPlanSignature = null ;
633663 }
634664 LOG .info ("MTMVTask refresh used snapshot: {}, mvDbName: {}, mvName: {}, taskId: {}" , partitionSnapshots ,
635665 mtmv .getDatabase ().getFullName (), mtmv .getName (), getTaskId ());
636666 }
637667
638- private void updateIvmPlanSignatureAfterFullRefreshIfNeeded () throws JobException {
639- if (IvmFailureReason .PLAN_SIGNATURE_MISMATCH .name ().equals (ivmFallbackReason )) {
640- refreshIvmPlanSignatureAfterFullRefresh ();
641- return ;
642- }
643- if (ivmFallbackPlanSignature == null ) {
644- return ;
645- }
646- IvmRefreshManager .updatePlanSignatureAfterFullRefresh (
647- mtmv , ivmFallbackPlanSignature , ivmFallbackPlanCanonicalString );
648- ivmFallbackPlanSignature = null ;
649- ivmFallbackPlanCanonicalString = null ;
650- }
651-
652- private void refreshIvmPlanSignatureAfterFullRefresh () throws JobException {
653- try {
654- ConnectContext ctx = MTMVPlanUtil .createMTMVContext (
655- mtmv , MTMVPlanUtil .DISABLE_RULES_WHEN_RUN_MTMV_TASK );
656- MTMVAnalyzeQueryInfo analyzeQueryInfo = MTMVPlanUtil .analyzeQueryWithSql (mtmv , ctx ,
657- Optional .of (IvmRewriteContext .normalize (mtmv )));
658- IvmPlanSignature currentSignature = analyzeQueryInfo .getIvmRewriteResult ().getPlanSignature ();
659- IvmRefreshManager .updatePlanSignatureAfterFullRefresh (
660- mtmv , currentSignature .getSha256 (), currentSignature .getCanonicalString ());
661- ivmFallbackPlanSignature = null ;
662- ivmFallbackPlanCanonicalString = null ;
663- } catch (Exception e ) {
664- throw new JobException ("Failed to rebuild IVM plan signature after full refresh, mv="
665- + mtmv .getName () + ", detail=" + e .getMessage (), e );
668+ private Map <BaseTableInfo , Set <Long >> collectPctResetPartitionIds (MTMVRefreshContext context ,
669+ Set <String > execPartitionNames ) throws AnalysisException {
670+ Map <BaseTableInfo , Set <Long >> resetPartitionIds = Maps .newHashMap ();
671+ for (String mvPartitionName : execPartitionNames ) {
672+ for (Entry <MTMVRelatedTableIf , Set <String >> entry
673+ : context .getByPartitionName (mvPartitionName ).entrySet ()) {
674+ if (!(entry .getKey () instanceof OlapTable )) {
675+ continue ;
676+ }
677+ OlapTable pctTable = (OlapTable ) entry .getKey ();
678+ Set <Long > partitionIds = resetPartitionIds .computeIfAbsent (
679+ new BaseTableInfo (pctTable ), key -> Sets .newHashSet ());
680+ for (String pctPartitionName : entry .getValue ()) {
681+ partitionIds .add (pctTable .getPartitionOrAnalysisException (pctPartitionName ).getId ());
682+ }
683+ }
666684 }
685+ return resetPartitionIds ;
667686 }
668687
669- private void executeWithRetry (Set <String > execPartitionNames , Map <TableIf , String > tableWithPartKey )
688+ private void executeWithRetry (Set <String > execPartitionNames , Map <TableIf , String > tableWithPartKey ,
689+ Optional <IvmRewriteContext > rewriteContext )
670690 throws Exception {
671691 int retryCount = 0 ;
672692 int retryTime = Config .max_query_retry_time ;
673693 retryTime = retryTime <= 0 ? 1 : retryTime + 1 ;
674694 Exception lastException = null ;
675695 while (retryCount < retryTime ) {
676696 try {
677- exec (execPartitionNames , tableWithPartKey );
697+ exec (execPartitionNames , tableWithPartKey , rewriteContext );
678698 break ; // Exit loop if execution is successful
679699 } catch (Exception e ) {
680700 if (!(Config .isCloudMode () && SystemInfoService .needRetryWithReplan (e .getMessage ()))) {
@@ -703,8 +723,8 @@ private void executeWithRetry(Set<String> execPartitionNames, Map<TableIf, Strin
703723 }
704724 }
705725
706- private void exec (Set <String > refreshPartitionNames ,
707- Map < TableIf , String > tableWithPartKey )
726+ private void exec (Set <String > refreshPartitionNames , Map < TableIf , String > tableWithPartKey ,
727+ Optional < IvmRewriteContext > rewriteContext )
708728 throws Exception {
709729 // Create MTMV context first so that new StatementContext() captures the
710730 // correct thread-local ConnectContext (with MTMV disabled rules, etc.).
@@ -719,9 +739,7 @@ private void exec(Set<String> refreshPartitionNames,
719739 mtmvCtx .setStatementContext (statementContext );
720740 statementContext .setConnectContext (mtmvCtx );
721741 statementContext .setExcludedTriggerTables (mtmv .getExcludedTriggerTables ());
722- if (mtmv .isIvm ()) {
723- statementContext .setIvmRewriteContext (Optional .of (IvmRewriteContext .full (mtmv )));
724- }
742+ statementContext .setIvmRewriteContext (rewriteContext );
725743 for (Entry <MvccTableInfo , MvccSnapshot > entry : snapshots .entrySet ()) {
726744 statementContext .setSnapshot (entry .getKey (), entry .getValue ());
727745 }
@@ -1005,7 +1023,7 @@ private Map<TableIf, String> getIncrementalTableMap() throws AnalysisException {
10051023 private MTMVTaskRefreshMode generateRefreshMode (List <String > needRefreshPartitionIds ) {
10061024 if (CollectionUtils .isEmpty (needRefreshPartitionIds )) {
10071025 return MTMVTaskRefreshMode .NOT_REFRESH ;
1008- } else if (needRefreshPartitionIds . size () == mtmv .getPartitionNames (). size ( )) {
1026+ } else if (Sets . newHashSet ( needRefreshPartitionIds ). equals ( mtmv .getPartitionNames ())) {
10091027 return MTMVTaskRefreshMode .COMPLETE ;
10101028 } else {
10111029 return MTMVTaskRefreshMode .PARTIAL ;
0 commit comments