Skip to content

Commit f69a307

Browse files
committed
[fix](fe) Retain deferred Hudi batch owners through statement cleanup
### What problem does this PR solve? Issue Number: N/A Related PR: #69298, #69986 Problem Summary: Hudi batch planning registers its filesystem-view owner directly as a statement resource. Closing the statement after GetFlightInfo therefore stops SplitAssignment and cancels its producer before a deferred Arrow Flight query can fetch splits during DoGet. The existing scan-level deferred cleanup guard cannot protect this independently registered owner. Register a statement-only cleanup wrapper that captures the query executor and checks its deferred flag at close time, while keeping the actual owner as an unconditional SplitAssignment resource for coordinator finalization and query cancellation. Preserve duplicate registration failure cleanup, task accounting, interruption and exact terminal lease release. The captured executor remains valid even if connection cleanup clears its current executor reference. ### Release note Keep deferred Hudi Arrow Flight batch scans alive until query finalization or cancellation. ### Check List (For Author) - Test: Unit tests and packaging - Reproduced the premature assignment stop through real HudiScanNode.startSplit and StatementContext.close before repair. - Added deferred, ordinary and absent-executor controls; verified registered source lifetime, forced scan stop, queued producer cancellation and one lease release. - All 90 selected FE tests passed with no failures, errors or skips using run-fe-ut.sh. - Full FE packaging and Checkstyle passed using build.sh --fe -j4; git diff --check and final source hashes passed. - No live Arrow Flight/Hudi external-service regression was run locally in this cycle. - Behavior changed: Yes. Deferred Hudi scans survive statement cleanup, while non-deferred cleanup and explicit query cancellation still stop the batch owner. - Does this need documentation: No
1 parent 7e8a562 commit f69a307

2 files changed

Lines changed: 163 additions & 7 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java‎

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@
5454
import org.apache.doris.planner.ScanContext;
5555
import org.apache.doris.qe.ConnectContext;
5656
import org.apache.doris.qe.SessionVariable;
57+
import org.apache.doris.qe.StmtExecutor;
5758
import org.apache.doris.spi.Split;
5859
import org.apache.doris.statistics.StatisticalType;
5960
import org.apache.doris.thrift.TExplainLevel;
@@ -716,21 +717,28 @@ public void startSplit(int numBackends) {
716717
ExecutorService scheduleExecutor = Env.getCurrentEnv().getExtMetaCacheMgr().getScheduleExecutor();
717718
Executor producerExecutor = Env.getCurrentEnv().getExtMetaCacheMgr().getFileListingExecutor();
718719
long startTime = System.currentTimeMillis();
719-
BatchFsViewOwner createdOwner = new BatchFsViewOwner(
720+
BatchFsViewOwner batchOwner = new BatchFsViewOwner(
720721
splitAssignment, fsViewLease, scheduleExecutor, producerExecutor);
721-
BatchFsViewOwner batchOwner = createdOwner;
722722
ConnectContext connectContext = ConnectContext.get();
723723
StatementContext statementContext = connectContext == null ? null : connectContext.getStatementContext();
724724
if (statementContext != null) {
725+
StmtExecutor executor = connectContext.getExecutor();
726+
Closeable statementCleanup = () -> {
727+
// GetFlightInfo ends the statement before the BE fetches deferred DoGet splits.
728+
// Capture this executor: processor cleanup can clear it from the connection context.
729+
if (executor == null || !executor.isDeferredForArrowFlight()) {
730+
batchOwner.close();
731+
}
732+
};
725733
try {
726-
batchOwner = statementContext.getOrRegisterStatementResource(
727-
batchFsViewResourceKey, () -> createdOwner);
728-
if (batchOwner != createdOwner) {
729-
createdOwner.finish();
734+
Closeable registered = statementContext.getOrRegisterStatementResource(
735+
batchFsViewResourceKey, () -> statementCleanup);
736+
if (registered != statementCleanup) {
737+
batchOwner.finish();
730738
throw new IllegalStateException("Hudi batch split owner was registered twice");
731739
}
732740
} catch (RuntimeException e) {
733-
createdOwner.finish();
741+
batchOwner.finish();
734742
throw e;
735743
}
736744
}

‎fe/fe-core/src/test/java/org/apache/doris/datasource/hudi/source/HudiBatchFsViewOwnerTest.java‎

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,17 +17,39 @@
1717

1818
package org.apache.doris.datasource.hudi.source;
1919

20+
import org.apache.doris.analysis.TupleDescriptor;
21+
import org.apache.doris.analysis.TupleId;
22+
import org.apache.doris.catalog.Env;
2023
import org.apache.doris.common.ThreadPoolManager;
24+
import org.apache.doris.common.security.authentication.ExecutionAuthenticator;
25+
import org.apache.doris.datasource.ExternalMetaCacheMgr;
2126
import org.apache.doris.datasource.SplitAssignment;
27+
import org.apache.doris.datasource.SplitSource;
28+
import org.apache.doris.datasource.SplitSourceManager;
29+
import org.apache.doris.datasource.hive.HMSExternalCatalog;
30+
import org.apache.doris.datasource.hive.HMSExternalTable;
31+
import org.apache.doris.datasource.hive.HivePartition;
32+
import org.apache.doris.datasource.hudi.HudiExternalMetaCache;
2233
import org.apache.doris.datasource.hudi.HudiFsViewCacheValue;
34+
import org.apache.doris.nereids.StatementContext;
35+
import org.apache.doris.planner.PlanNodeId;
36+
import org.apache.doris.planner.ScanContext;
37+
import org.apache.doris.planner.ScanNode;
38+
import org.apache.doris.qe.ConnectContext;
39+
import org.apache.doris.qe.SessionVariable;
40+
import org.apache.doris.qe.StmtExecutor;
41+
import org.apache.doris.system.Backend;
2342

2443
import org.junit.jupiter.api.Assertions;
2544
import org.junit.jupiter.api.Test;
45+
import org.mockito.MockedStatic;
2646
import org.mockito.Mockito;
2747

48+
import java.lang.reflect.Field;
2849
import java.util.ArrayList;
2950
import java.util.Collections;
3051
import java.util.List;
52+
import java.util.Optional;
3153
import java.util.concurrent.CancellationException;
3254
import java.util.concurrent.CountDownLatch;
3355
import java.util.concurrent.Executor;
@@ -42,6 +64,132 @@
4264

4365
class HudiBatchFsViewOwnerTest {
4466

67+
@Test
68+
void deferredBatchSourceSurvivesStatementCloseUntilCoordinatorStopsIt() throws Exception {
69+
try (BatchScanFixture f = new BatchScanFixture()) {
70+
f.node.startSplit(1);
71+
Assertions.assertEquals(1, f.submitted.size());
72+
SplitSource source = new SplitSource(new Backend(1, "localhost", 0), f.assignment, 1000);
73+
f.manager.registerSplitSource(source);
74+
// Deferral is selected after planning. Processor cleanup can also clear the context's executor.
75+
Mockito.when(f.executor.isDeferredForArrowFlight()).thenReturn(true);
76+
f.context.setExecutor(null);
77+
f.statement.close();
78+
79+
Assertions.assertFalse(f.assignment.isStop(), "DoGet still needs the deferred batch source");
80+
Assertions.assertSame(source, f.manager.getSplitSource(source.getUniqueId()));
81+
Assertions.assertFalse(f.producer().isCancelled());
82+
Mockito.verify(f.lease, Mockito.never()).close();
83+
84+
f.node.stop();
85+
86+
Assertions.assertTrue(f.assignment.isStop());
87+
Assertions.assertNull(f.manager.getSplitSource(source.getUniqueId()));
88+
Assertions.assertTrue(f.producer().isCancelled());
89+
Mockito.verify(f.lease).close();
90+
}
91+
}
92+
93+
@Test
94+
void ordinaryBatchStatementCloseStillCancelsQueuedPlanning() throws Exception {
95+
try (BatchScanFixture f = new BatchScanFixture()) {
96+
f.node.startSplit(1);
97+
f.statement.close();
98+
99+
Assertions.assertTrue(f.assignment.isStop());
100+
Assertions.assertTrue(f.producer().isCancelled());
101+
Mockito.verify(f.lease).close();
102+
}
103+
}
104+
105+
@Test
106+
void abandonedBatchWithoutExecutorStillStopsAtStatementClose() throws Exception {
107+
try (BatchScanFixture f = new BatchScanFixture()) {
108+
f.context.setExecutor(null);
109+
f.node.startSplit(1);
110+
f.statement.close();
111+
112+
Assertions.assertTrue(f.assignment.isStop());
113+
Assertions.assertTrue(f.producer().isCancelled());
114+
Mockito.verify(f.lease).close();
115+
}
116+
}
117+
118+
private static class BatchScanFixture implements AutoCloseable {
119+
private final ConnectContext previous = ConnectContext.get();
120+
private final ConnectContext context = new ConnectContext();
121+
private final StatementContext statement = new StatementContext(context, null);
122+
private final StmtExecutor executor = Mockito.mock(StmtExecutor.class);
123+
private final SplitAssignment assignment = new SplitAssignment(
124+
null, null, null, Collections.emptyMap(), Collections.emptyList(), false);
125+
private final SplitSourceManager manager = new SplitSourceManager();
126+
private final HudiFsViewCacheValue.Lease lease = Mockito.mock(HudiFsViewCacheValue.Lease.class);
127+
private final List<Runnable> submitted = new ArrayList<>();
128+
private final MockedStatic<Env> mockedEnv;
129+
private final HudiScanNode node;
130+
131+
BatchScanFixture() throws Exception {
132+
context.setStatementContext(statement);
133+
context.setExecutor(executor);
134+
context.setThreadLocalInfo();
135+
Env env = Mockito.mock(Env.class);
136+
ExternalMetaCacheMgr cache = Mockito.mock(ExternalMetaCacheMgr.class);
137+
ExecutorService worker = Mockito.mock(ExecutorService.class);
138+
Mockito.doAnswer(call -> {
139+
submitted.add(call.getArgument(0));
140+
return null;
141+
}).when(worker).execute(Mockito.any(Runnable.class));
142+
Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cache);
143+
Mockito.when(env.getSplitSourceManager()).thenReturn(manager);
144+
Mockito.when(cache.getScheduleExecutor()).thenReturn(worker);
145+
Mockito.when(cache.getFileListingExecutor()).thenReturn(worker);
146+
mockedEnv = Mockito.mockStatic(Env.class);
147+
mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
148+
HMSExternalTable table = Mockito.mock(HMSExternalTable.class);
149+
HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class);
150+
HMSExternalCatalog.HudiScanRuntimeContext runtime =
151+
Mockito.mock(HMSExternalCatalog.HudiScanRuntimeContext.class);
152+
HudiExternalMetaCache.FsViewGeneration generation =
153+
Mockito.mock(HudiExternalMetaCache.FsViewGeneration.class);
154+
Mockito.when(table.getCatalog()).thenReturn(catalog);
155+
Mockito.when(catalog.getHudiScanRuntimeContext()).thenReturn(runtime);
156+
Mockito.when(runtime.getAuthenticator()).thenReturn(new ExecutionAuthenticator() {});
157+
Mockito.when(runtime.getFsViewGeneration()).thenReturn(generation);
158+
Mockito.when(generation.acquireFsView(Mockito.any())).thenReturn(lease);
159+
TupleDescriptor desc = new TupleDescriptor(new TupleId(0));
160+
desc.setTable(table);
161+
node = new HudiScanNode(new PlanNodeId(0), desc, false, Optional.empty(), Optional.empty(),
162+
new SessionVariable(), null, ScanContext.EMPTY);
163+
setField(node, HudiScanNode.class, "prunedPartitions",
164+
Collections.singletonList(Mockito.mock(HivePartition.class)));
165+
setField(node, ScanNode.class, "splitAssignment", assignment);
166+
}
167+
168+
private static void setField(Object target, Class<?> owner, String name, Object value) throws Exception {
169+
Field field = owner.getDeclaredField(name);
170+
field.setAccessible(true);
171+
field.set(target, value);
172+
}
173+
174+
HudiScanNode.TerminalTask producer() {
175+
return (HudiScanNode.TerminalTask) submitted.get(0);
176+
}
177+
178+
@Override
179+
public void close() {
180+
try {
181+
node.stop();
182+
statement.close();
183+
} finally {
184+
mockedEnv.close();
185+
ConnectContext.remove();
186+
if (previous != null) {
187+
previous.setThreadLocalInfo();
188+
}
189+
}
190+
}
191+
}
192+
45193
@Test
46194
void statementCloseReturnsWhileRunningTaskKeepsLeasePinned() throws Exception {
47195
SplitAssignment assignment = Mockito.mock(SplitAssignment.class);

0 commit comments

Comments
 (0)