Skip to content

Commit d4a974d

Browse files
committed
[test](arrow-flight) Add FE unit tests for deferred-coordinator resource cleanup
The fix for apache#62259 keeps an Arrow Flight query's coordinator alive across GetFlightInfo -> DoGet and releases it later via deferred executors held on the ConnectContext. These tests pin the leak-prevention contract of that deferred cleanup so the coordinator (and the external-table batch SplitSource and query queue slot it holds, plus the query registration) cannot leak: - ConnectContextTest: closeFlightSqlDeferredExecutors() finalizes every deferred executor, clears the list (idempotent, no double-close, no retain), and still finalizes the rest when one executor's finalize throws. - FlightSqlConnectPoolMgrTest (new): unregisterConnection() finalizes the deferred executors on connection teardown (idle/query timeout, bearer token expiry, explicit CloseSession), even for a connection that was never registered, so an abandoned connection cannot leak the coordinator. - StmtExecutorTest: finalizeArrowFlightQuery() still unregisters the query (releases the registration) when coord.close() throws (the try/finally). Each test was verified to fail when its corresponding cleanup is removed.
1 parent e57f3d6 commit d4a974d

3 files changed

Lines changed: 156 additions & 0 deletions

File tree

‎fe/fe-core/src/test/java/org/apache/doris/qe/ConnectContextTest.java‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -753,4 +753,62 @@ public void testConnectAttributesSetNull() {
753753
Assert.assertNotNull(ctx.getConnectAttributes());
754754
Assert.assertTrue(ctx.getConnectAttributes().isEmpty());
755755
}
756+
757+
// Arrow Flight SQL keeps a query's coordinator alive across GetFlightInfo -> DoGet (see #62259).
758+
// closeFlightSqlDeferredExecutors() is the single place that releases those deferred coordinators
759+
// (and with them the external-table batch SplitSource and the query queue slot). The following
760+
// tests pin the leak-prevention contract of that method: every deferred executor is finalized,
761+
// the list is cleared so nothing is finalized twice or retained, and one failing executor does
762+
// not strand the others' resources.
763+
764+
@Test
765+
public void testCloseFlightSqlDeferredExecutorsFinalizesEachExecutor() {
766+
ConnectContext ctx = new ConnectContext();
767+
StmtExecutor deferred1 = Mockito.mock(StmtExecutor.class);
768+
StmtExecutor deferred2 = Mockito.mock(StmtExecutor.class);
769+
ctx.addFlightSqlDeferredExecutor(deferred1);
770+
ctx.addFlightSqlDeferredExecutor(deferred2);
771+
772+
ctx.closeFlightSqlDeferredExecutors();
773+
774+
// Both deferred coordinators must be finalized, otherwise their SplitSource and query queue
775+
// slot leak after the DoGet phase.
776+
Mockito.verify(deferred1).finalizeArrowFlightQuery();
777+
Mockito.verify(deferred2).finalizeArrowFlightQuery();
778+
}
779+
780+
@Test
781+
public void testCloseFlightSqlDeferredExecutorsClearsListSoSecondCallIsNoOp() {
782+
ConnectContext ctx = new ConnectContext();
783+
StmtExecutor deferred = Mockito.mock(StmtExecutor.class);
784+
ctx.addFlightSqlDeferredExecutor(deferred);
785+
786+
// More than one teardown path can fire for the same connection (e.g. the next query cleans
787+
// up, then the connection is later torn down). The list must be cleared after the first
788+
// call so the executor is finalized exactly once and is not retained (leaked) afterwards.
789+
ctx.closeFlightSqlDeferredExecutors();
790+
ctx.closeFlightSqlDeferredExecutors();
791+
792+
Mockito.verify(deferred, Mockito.times(1)).finalizeArrowFlightQuery();
793+
}
794+
795+
@Test
796+
public void testCloseFlightSqlDeferredExecutorsFinalizesRemainingWhenOneFails() {
797+
ConnectContext ctx = new ConnectContext();
798+
StmtExecutor failing = Mockito.mock(StmtExecutor.class);
799+
StmtExecutor healthy = Mockito.mock(StmtExecutor.class);
800+
Mockito.doThrow(new RuntimeException("finalize failed")).when(failing).finalizeArrowFlightQuery();
801+
ctx.addFlightSqlDeferredExecutor(failing);
802+
ctx.addFlightSqlDeferredExecutor(healthy);
803+
804+
// A single bad coordinator must not abort the cleanup: the call must not throw, and the
805+
// healthy executor must still be finalized so its resources are released.
806+
ctx.closeFlightSqlDeferredExecutors();
807+
Mockito.verify(healthy).finalizeArrowFlightQuery();
808+
809+
// The list is cleared up front, so neither executor is reprocessed on a later teardown.
810+
ctx.closeFlightSqlDeferredExecutors();
811+
Mockito.verify(failing, Mockito.times(1)).finalizeArrowFlightQuery();
812+
Mockito.verify(healthy, Mockito.times(1)).finalizeArrowFlightQuery();
813+
}
756814
}

‎fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,8 @@
2929
import org.apache.doris.planner.ResultFileSink;
3030
import org.apache.doris.qe.CommonResultSet.CommonResultSetMetaData;
3131
import org.apache.doris.qe.ConnectContext.ConnectType;
32+
import org.apache.doris.thrift.TQueryOptions;
33+
import org.apache.doris.thrift.TUniqueId;
3234
import org.apache.doris.utframe.TestWithFeService;
3335

3436
import com.google.common.collect.Lists;
@@ -71,6 +73,40 @@ public void testShowNull() throws Exception {
7173
Assert.assertEquals(QueryState.MysqlStateType.OK, connectContext.getState().getStateType());
7274
}
7375

76+
// Arrow Flight SQL keeps a query's coordinator alive across GetFlightInfo -> DoGet (see #62259);
77+
// it is released later by finalizeArrowFlightQuery(), which closes the coordinator and then
78+
// unregisters the query. The close and the unregister must be independent: if coord.close()
79+
// throws, the query registration must still be released (the try/finally), otherwise the query
80+
// leaks in QeProcessorImpl forever. The thrown error is expected to propagate to the caller
81+
// (ConnectContext.closeFlightSqlDeferredExecutors), which catches and logs it.
82+
@Test
83+
public void testFinalizeArrowFlightQueryUnregistersQueryEvenIfCoordCloseThrows() throws Exception {
84+
StmtExecutor stmtExecutor = new StmtExecutor(connectContext, "");
85+
TUniqueId queryId = new TUniqueId(0x6226259L, 0x62259L);
86+
connectContext.setQueryId(queryId);
87+
88+
Coordinator coord = Mockito.mock(Coordinator.class);
89+
Mockito.when(coord.getQueryOptions()).thenReturn(new TQueryOptions());
90+
Mockito.doThrow(new RuntimeException("coord close failed")).when(coord).close();
91+
stmtExecutor.setCoord(coord);
92+
93+
// Simulate the in-flight query whose results DoGet is still pulling.
94+
QeProcessorImpl.INSTANCE.registerQuery(queryId, new QeProcessorImpl.QueryInfo(coord));
95+
Assert.assertNotNull(QeProcessorImpl.INSTANCE.getCoordinator(queryId));
96+
97+
try {
98+
stmtExecutor.finalizeArrowFlightQuery();
99+
Assert.fail("expected coord.close() failure to propagate after the query is unregistered");
100+
} catch (RuntimeException e) {
101+
Assert.assertEquals("coord close failed", e.getMessage());
102+
}
103+
104+
// The coordinator close was attempted (releases SplitSource + query queue slot) ...
105+
Mockito.verify(coord).close();
106+
// ... and despite it failing, the query registration was still released (no leak).
107+
Assert.assertNull(QeProcessorImpl.INSTANCE.getCoordinator(queryId));
108+
}
109+
74110
@Test
75111
public void testKill() throws Exception {
76112
StmtExecutor stmtExecutor = new StmtExecutor(connectContext, "");
Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.service.arrowflight.sessions;
19+
20+
import org.apache.doris.qe.ConnectContext;
21+
22+
import org.junit.Assert;
23+
import org.junit.Test;
24+
import org.mockito.Mockito;
25+
26+
public class FlightSqlConnectPoolMgrTest {
27+
28+
// Arrow Flight SQL keeps a query's coordinator alive across GetFlightInfo -> DoGet (see #62259).
29+
// unregisterConnection() is the catch-all teardown path: idle/query timeout, bearer token expiry
30+
// and explicit CloseSession all reach here. It must finalize the deferred coordinators so an
31+
// abandoned connection cannot leak them (the external-table batch SplitSource and the query
32+
// queue slot the coordinator holds).
33+
@Test
34+
public void testUnregisterConnectionFinalizesDeferredExecutors() {
35+
FlightSqlConnectPoolMgr poolMgr = new FlightSqlConnectPoolMgr(100);
36+
ConnectContext ctx = Mockito.mock(ConnectContext.class);
37+
38+
poolMgr.unregisterConnection(ctx);
39+
40+
// The deferred coordinators must be released on teardown even though this connection was
41+
// never registered in the pool (an abandoned connection is still cleaned up, not leaked).
42+
Mockito.verify(ctx).closeFlightSqlDeferredExecutors();
43+
}
44+
45+
// Cleanup must run before the connection bookkeeping (closeTxn / map removal), so that a failure
46+
// there cannot strand the deferred coordinators. Verify the deferred executors are finalized
47+
// even when the context is the one stored in the pool.
48+
@Test
49+
public void testUnregisterRegisteredConnectionFinalizesDeferredExecutors() {
50+
FlightSqlConnectPoolMgr poolMgr = new FlightSqlConnectPoolMgr(100);
51+
ConnectContext ctx = Mockito.mock(ConnectContext.class);
52+
Mockito.when(ctx.getConnectionId()).thenReturn(7);
53+
Mockito.when(ctx.getConnectType()).thenReturn(ConnectContext.ConnectType.ARROW_FLIGHT_SQL);
54+
Mockito.when(ctx.getPeerIdentity()).thenReturn("token-7");
55+
poolMgr.getConnectionMap().put(7, ctx);
56+
57+
poolMgr.unregisterConnection(ctx);
58+
59+
Mockito.verify(ctx).closeFlightSqlDeferredExecutors();
60+
Assert.assertNull(poolMgr.getConnectionMap().get(7));
61+
}
62+
}

0 commit comments

Comments
 (0)