|
27 | 27 | import org.apache.doris.catalog.Partition; |
28 | 28 | import org.apache.doris.catalog.info.TableNameInfo; |
29 | 29 | import org.apache.doris.catalog.stream.OlapTableStream; |
| 30 | +import org.apache.doris.changelog.ChangelogUtils; |
30 | 31 | import org.apache.doris.common.AnalysisException; |
31 | 32 | import org.apache.doris.common.Config; |
32 | 33 | import org.apache.doris.common.DdlException; |
@@ -1775,6 +1776,91 @@ public void testExecuteIvmAttemptFallsBackToCompleteForBrokenBaseline() throws E |
1775 | 1776 | } |
1776 | 1777 | } |
1777 | 1778 |
|
| 1779 | + /** |
| 1780 | + * The rebuild an incremental attempt runs first, for the MV partitions that need one, reads the tables the MV |
| 1781 | + * does not partition by through their streams at the offsets. An offset changelog.ttl_seconds has expired stops |
| 1782 | + * it the way it stops the delta, so the answer is the same: a request that may fall back goes to COMPLETE, the |
| 1783 | + * only attempt that resets the stream baselines. |
| 1784 | + */ |
| 1785 | + @Test |
| 1786 | + public void testExpiredOffsetInTheRebuildBeforeTheDeltaFallsBackToComplete() throws Exception { |
| 1787 | + MTMVTask task = new MTMVTask(mtmv, relation, MTMVTaskContext.of(MTMVTaskTriggerMode.MANUAL, null, |
| 1788 | + RefreshMode.INCREMENTAL, true, null)); |
| 1789 | + |
| 1790 | + Object result = runIvmAttemptWhoseRebuildReadsAnExpiredOffset(task); |
| 1791 | + |
| 1792 | + Assertions.assertEquals("FALLBACK_TO_COMPLETE", result.toString()); |
| 1793 | + Assertions.assertEquals(IvmFailureReason.CHANGELOG_BROKEN.name(), |
| 1794 | + Deencapsulation.getField(task, "ivmFallbackReason")); |
| 1795 | + } |
| 1796 | + |
| 1797 | + /** A request that may not fall back fails instead, with the reason it would have fallen back for. */ |
| 1798 | + @Test |
| 1799 | + public void testExpiredOffsetInTheRebuildBeforeTheDeltaFailsAStrictRequest() { |
| 1800 | + MTMVTask task = new MTMVTask(mtmv, relation, MTMVTaskContext.of(MTMVTaskTriggerMode.MANUAL, null, |
| 1801 | + RefreshMode.INCREMENTAL, false, null)); |
| 1802 | + |
| 1803 | + JobException failure = Assertions.assertThrows(JobException.class, |
| 1804 | + () -> runIvmAttemptWhoseRebuildReadsAnExpiredOffset(task)); |
| 1805 | + |
| 1806 | + Assertions.assertTrue(failure.getMessage().contains("reason=" + IvmFailureReason.CHANGELOG_BROKEN)); |
| 1807 | + Assertions.assertTrue(failure.getMessage().contains(ChangelogUtils.CHANGELOG_OFFSET_EXPIRED)); |
| 1808 | + Assertions.assertEquals(IvmFailureReason.CHANGELOG_BROKEN.name(), |
| 1809 | + Deencapsulation.getField(task, "ivmFallbackReason")); |
| 1810 | + } |
| 1811 | + |
| 1812 | + /** |
| 1813 | + * Runs the incremental attempt with p1 needing a rebuild and p2 caught up, the rebuild's read refused the way |
| 1814 | + * planning refuses a MIN_DELTA read from an expired offset. The delta must not be reached. |
| 1815 | + */ |
| 1816 | + private Object runIvmAttemptWhoseRebuildReadsAnExpiredOffset(MTMVTask task) throws Exception { |
| 1817 | + Mockito.when(mtmv.isIvm()).thenReturn(true); |
| 1818 | + Mockito.when(mtmv.getName()).thenReturn("test_mv"); |
| 1819 | + Mockito.when(mtmv.hasRefreshSnapshot()).thenReturn(true); |
| 1820 | + Mockito.when(mtmv.getRefreshPartitionNum()).thenReturn(1); |
| 1821 | + Mockito.when(mtmvPartitionInfo.getPctInfos()).thenReturn(Collections.emptyList()); |
| 1822 | + Mockito.when(mtmv.getPartitionNames()).thenReturn(Sets.newHashSet(poneName, ptwoName)); |
| 1823 | + Mockito.when(mtmv.getPartitionStates()).thenReturn(Maps.newHashMap(Map.of( |
| 1824 | + poneName, new MTMVPartitionState(1, 2), |
| 1825 | + ptwoName, new MTMVPartitionState(2, 2)))); |
| 1826 | + MTMVRefreshContext refreshContext = Mockito.mock(MTMVRefreshContext.class); |
| 1827 | + Mockito.when(refreshContext.getByPartitionName(Mockito.anyString())).thenReturn(Collections.emptyMap()); |
| 1828 | + mtmvPartitionUtilStatic.when(() -> MTMVPartitionUtil.generatePartitionSnapshots( |
| 1829 | + Mockito.same(refreshContext), Mockito.anySet(), Mockito.anySet())) |
| 1830 | + .thenReturn(Collections.emptyMap()); |
| 1831 | + RuntimeException expired = Assertions.assertThrows(RuntimeException.class, |
| 1832 | + () -> ChangelogUtils.effectiveStartTso(99L, 100L, true)); |
| 1833 | + |
| 1834 | + ConnectContext mtmvCtx = new ConnectContext(); |
| 1835 | + mtmvCtx.setQueryId(new TUniqueId(1L, 2L)); |
| 1836 | + mtmvCtx.setThreadLocalInfo(); |
| 1837 | + UpdateMvByPartitionCommand command = Mockito.mock(UpdateMvByPartitionCommand.class); |
| 1838 | + try (MockedStatic<MTMVPlanUtil> mtmvPlanUtilStatic = Mockito.mockStatic(MTMVPlanUtil.class); |
| 1839 | + MockedStatic<UpdateMvByPartitionCommand> updateMvStatic |
| 1840 | + = Mockito.mockStatic(UpdateMvByPartitionCommand.class); |
| 1841 | + MockedConstruction<IvmIncrRefreshManager> delta |
| 1842 | + = Mockito.mockConstruction(IvmIncrRefreshManager.class)) { |
| 1843 | + mtmvPlanUtilStatic.when(() -> MTMVPlanUtil.createMTMVContext(Mockito.eq(mtmv), Mockito.anyList())) |
| 1844 | + .thenReturn(mtmvCtx); |
| 1845 | + updateMvStatic.when(() -> UpdateMvByPartitionCommand.from( |
| 1846 | + Mockito.eq(mtmv), Mockito.anySet(), Mockito.anyMap(), Mockito.any(StatementContext.class))) |
| 1847 | + .thenReturn(command); |
| 1848 | + mtmvPlanUtilStatic.when(() -> MTMVPlanUtil.executeCommand( |
| 1849 | + Mockito.eq(mtmvCtx), Mockito.eq(command), Mockito.any(StatementContext.class), |
| 1850 | + Mockito.anyString(), Mockito.any(Consumer.class))).thenThrow(expired); |
| 1851 | + |
| 1852 | + Object request = Deencapsulation.invoke(task, "resolveRefreshRequest"); |
| 1853 | + try { |
| 1854 | + return Deencapsulation.invoke(task, "executeIvmAttempt", refreshContext, request, mtmvCtx, |
| 1855 | + Lists.newArrayList()); |
| 1856 | + } finally { |
| 1857 | + Assertions.assertTrue(delta.constructed().isEmpty()); |
| 1858 | + } |
| 1859 | + } finally { |
| 1860 | + ConnectContext.remove(); |
| 1861 | + } |
| 1862 | + } |
| 1863 | + |
1778 | 1864 |
|
1779 | 1865 |
|
1780 | 1866 |
|
|
0 commit comments