|
17 | 17 |
|
18 | 18 | package org.apache.doris.datasource.hudi.source; |
19 | 19 |
|
| 20 | +import org.apache.doris.analysis.TupleDescriptor; |
| 21 | +import org.apache.doris.analysis.TupleId; |
| 22 | +import org.apache.doris.catalog.Env; |
20 | 23 | import org.apache.doris.common.ThreadPoolManager; |
| 24 | +import org.apache.doris.common.security.authentication.ExecutionAuthenticator; |
| 25 | +import org.apache.doris.datasource.ExternalMetaCacheMgr; |
21 | 26 | import org.apache.doris.datasource.SplitAssignment; |
| 27 | +import org.apache.doris.datasource.SplitSource; |
| 28 | +import org.apache.doris.datasource.SplitSourceManager; |
| 29 | +import org.apache.doris.datasource.hive.HMSExternalCatalog; |
| 30 | +import org.apache.doris.datasource.hive.HMSExternalTable; |
| 31 | +import org.apache.doris.datasource.hive.HivePartition; |
| 32 | +import org.apache.doris.datasource.hudi.HudiExternalMetaCache; |
22 | 33 | import org.apache.doris.datasource.hudi.HudiFsViewCacheValue; |
| 34 | +import org.apache.doris.nereids.StatementContext; |
| 35 | +import org.apache.doris.planner.PlanNodeId; |
| 36 | +import org.apache.doris.planner.ScanContext; |
| 37 | +import org.apache.doris.planner.ScanNode; |
| 38 | +import org.apache.doris.qe.ConnectContext; |
| 39 | +import org.apache.doris.qe.SessionVariable; |
| 40 | +import org.apache.doris.qe.StmtExecutor; |
| 41 | +import org.apache.doris.system.Backend; |
23 | 42 |
|
24 | 43 | import org.junit.jupiter.api.Assertions; |
25 | 44 | import org.junit.jupiter.api.Test; |
| 45 | +import org.mockito.MockedStatic; |
26 | 46 | import org.mockito.Mockito; |
27 | 47 |
|
| 48 | +import java.lang.reflect.Field; |
28 | 49 | import java.util.ArrayList; |
29 | 50 | import java.util.Collections; |
30 | 51 | import java.util.List; |
| 52 | +import java.util.Optional; |
31 | 53 | import java.util.concurrent.CancellationException; |
32 | 54 | import java.util.concurrent.CountDownLatch; |
33 | 55 | import java.util.concurrent.Executor; |
|
42 | 64 |
|
43 | 65 | class HudiBatchFsViewOwnerTest { |
44 | 66 |
|
| 67 | + @Test |
| 68 | + void deferredBatchSourceSurvivesStatementCloseUntilCoordinatorStopsIt() throws Exception { |
| 69 | + try (BatchScanFixture f = new BatchScanFixture()) { |
| 70 | + f.node.startSplit(1); |
| 71 | + Assertions.assertEquals(1, f.submitted.size()); |
| 72 | + SplitSource source = new SplitSource(new Backend(1, "localhost", 0), f.assignment, 1000); |
| 73 | + f.manager.registerSplitSource(source); |
| 74 | + // Deferral is selected after planning. Processor cleanup can also clear the context's executor. |
| 75 | + Mockito.when(f.executor.isDeferredForArrowFlight()).thenReturn(true); |
| 76 | + f.context.setExecutor(null); |
| 77 | + f.statement.close(); |
| 78 | + |
| 79 | + Assertions.assertFalse(f.assignment.isStop(), "DoGet still needs the deferred batch source"); |
| 80 | + Assertions.assertSame(source, f.manager.getSplitSource(source.getUniqueId())); |
| 81 | + Assertions.assertFalse(f.producer().isCancelled()); |
| 82 | + Mockito.verify(f.lease, Mockito.never()).close(); |
| 83 | + |
| 84 | + f.node.stop(); |
| 85 | + |
| 86 | + Assertions.assertTrue(f.assignment.isStop()); |
| 87 | + Assertions.assertNull(f.manager.getSplitSource(source.getUniqueId())); |
| 88 | + Assertions.assertTrue(f.producer().isCancelled()); |
| 89 | + Mockito.verify(f.lease).close(); |
| 90 | + } |
| 91 | + } |
| 92 | + |
| 93 | + @Test |
| 94 | + void ordinaryBatchStatementCloseStillCancelsQueuedPlanning() throws Exception { |
| 95 | + try (BatchScanFixture f = new BatchScanFixture()) { |
| 96 | + f.node.startSplit(1); |
| 97 | + f.statement.close(); |
| 98 | + |
| 99 | + Assertions.assertTrue(f.assignment.isStop()); |
| 100 | + Assertions.assertTrue(f.producer().isCancelled()); |
| 101 | + Mockito.verify(f.lease).close(); |
| 102 | + } |
| 103 | + } |
| 104 | + |
| 105 | + @Test |
| 106 | + void abandonedBatchWithoutExecutorStillStopsAtStatementClose() throws Exception { |
| 107 | + try (BatchScanFixture f = new BatchScanFixture()) { |
| 108 | + f.context.setExecutor(null); |
| 109 | + f.node.startSplit(1); |
| 110 | + f.statement.close(); |
| 111 | + |
| 112 | + Assertions.assertTrue(f.assignment.isStop()); |
| 113 | + Assertions.assertTrue(f.producer().isCancelled()); |
| 114 | + Mockito.verify(f.lease).close(); |
| 115 | + } |
| 116 | + } |
| 117 | + |
| 118 | + private static class BatchScanFixture implements AutoCloseable { |
| 119 | + private final ConnectContext previous = ConnectContext.get(); |
| 120 | + private final ConnectContext context = new ConnectContext(); |
| 121 | + private final StatementContext statement = new StatementContext(context, null); |
| 122 | + private final StmtExecutor executor = Mockito.mock(StmtExecutor.class); |
| 123 | + private final SplitAssignment assignment = new SplitAssignment( |
| 124 | + null, null, null, Collections.emptyMap(), Collections.emptyList(), false); |
| 125 | + private final SplitSourceManager manager = new SplitSourceManager(); |
| 126 | + private final HudiFsViewCacheValue.Lease lease = Mockito.mock(HudiFsViewCacheValue.Lease.class); |
| 127 | + private final List<Runnable> submitted = new ArrayList<>(); |
| 128 | + private final MockedStatic<Env> mockedEnv; |
| 129 | + private final HudiScanNode node; |
| 130 | + |
| 131 | + BatchScanFixture() throws Exception { |
| 132 | + context.setStatementContext(statement); |
| 133 | + context.setExecutor(executor); |
| 134 | + context.setThreadLocalInfo(); |
| 135 | + Env env = Mockito.mock(Env.class); |
| 136 | + ExternalMetaCacheMgr cache = Mockito.mock(ExternalMetaCacheMgr.class); |
| 137 | + ExecutorService worker = Mockito.mock(ExecutorService.class); |
| 138 | + Mockito.doAnswer(call -> { |
| 139 | + submitted.add(call.getArgument(0)); |
| 140 | + return null; |
| 141 | + }).when(worker).execute(Mockito.any(Runnable.class)); |
| 142 | + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cache); |
| 143 | + Mockito.when(env.getSplitSourceManager()).thenReturn(manager); |
| 144 | + Mockito.when(cache.getScheduleExecutor()).thenReturn(worker); |
| 145 | + Mockito.when(cache.getFileListingExecutor()).thenReturn(worker); |
| 146 | + mockedEnv = Mockito.mockStatic(Env.class); |
| 147 | + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); |
| 148 | + HMSExternalTable table = Mockito.mock(HMSExternalTable.class); |
| 149 | + HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class); |
| 150 | + HMSExternalCatalog.HudiScanRuntimeContext runtime = |
| 151 | + Mockito.mock(HMSExternalCatalog.HudiScanRuntimeContext.class); |
| 152 | + HudiExternalMetaCache.FsViewGeneration generation = |
| 153 | + Mockito.mock(HudiExternalMetaCache.FsViewGeneration.class); |
| 154 | + Mockito.when(table.getCatalog()).thenReturn(catalog); |
| 155 | + Mockito.when(catalog.getHudiScanRuntimeContext()).thenReturn(runtime); |
| 156 | + Mockito.when(runtime.getAuthenticator()).thenReturn(new ExecutionAuthenticator() {}); |
| 157 | + Mockito.when(runtime.getFsViewGeneration()).thenReturn(generation); |
| 158 | + Mockito.when(generation.acquireFsView(Mockito.any())).thenReturn(lease); |
| 159 | + TupleDescriptor desc = new TupleDescriptor(new TupleId(0)); |
| 160 | + desc.setTable(table); |
| 161 | + node = new HudiScanNode(new PlanNodeId(0), desc, false, Optional.empty(), Optional.empty(), |
| 162 | + new SessionVariable(), null, ScanContext.EMPTY); |
| 163 | + setField(node, HudiScanNode.class, "prunedPartitions", |
| 164 | + Collections.singletonList(Mockito.mock(HivePartition.class))); |
| 165 | + setField(node, ScanNode.class, "splitAssignment", assignment); |
| 166 | + } |
| 167 | + |
| 168 | + private static void setField(Object target, Class<?> owner, String name, Object value) throws Exception { |
| 169 | + Field field = owner.getDeclaredField(name); |
| 170 | + field.setAccessible(true); |
| 171 | + field.set(target, value); |
| 172 | + } |
| 173 | + |
| 174 | + HudiScanNode.TerminalTask producer() { |
| 175 | + return (HudiScanNode.TerminalTask) submitted.get(0); |
| 176 | + } |
| 177 | + |
| 178 | + @Override |
| 179 | + public void close() { |
| 180 | + try { |
| 181 | + node.stop(); |
| 182 | + statement.close(); |
| 183 | + } finally { |
| 184 | + mockedEnv.close(); |
| 185 | + ConnectContext.remove(); |
| 186 | + if (previous != null) { |
| 187 | + previous.setThreadLocalInfo(); |
| 188 | + } |
| 189 | + } |
| 190 | + } |
| 191 | + } |
| 192 | + |
45 | 193 | @Test |
46 | 194 | void statementCloseReturnsWhileRunningTaskKeepsLeasePinned() throws Exception { |
47 | 195 | SplitAssignment assignment = Mockito.mock(SplitAssignment.class); |
|
0 commit comments