Skip to content

Commit 41a79a6

Browse files
morningmanclaude
andcommitted
[fix](arrow-flight) Finalize deferred coordinator when GetFlightInfo fails after deferral
For an Arrow Flight external-table scan, executeAndSendResult() registers the query's coordinator as a deferred executor on the ConnectContext (kept alive across GetFlightInfo -> DoGet, see #62259) right after submitting it to the BE. GetFlightInfo then still has to fetch the Arrow schema from the BE. If that fetch fails (timeout / non-OK / empty / mismatched schema / RPC error), FlightSqlConnectProcessor.close() skips the deferred executor and the outer catch only rethrows. No FlightInfo is returned, so no DoGet will ever pull the results, yet the coordinator (its external-table batch SplitSource, the query queue slot and the query registration) stays alive until the next query starts or the connection is torn down. Finalize the deferred coordinator on the GetFlightInfo error path: executeQueryStatement's catch now calls closeFlightSqlDeferredExecutors(). At that point the list holds only this failed query's coordinator (the previous one was already finalized at the top of the method), and this covers every post-deferral failure, not just the schema fetch. Add DorisFlightSqlProducerTest.testGetFlightInfoFinalizesDeferredExecutorWhen SchemaFetchFails, which defers a coordinator then fails the schema fetch and asserts the deferred executor is finalized (verified to fail without the fix). Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
1 parent cd50205 commit 41a79a6

2 files changed

Lines changed: 104 additions & 0 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducer.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -284,6 +284,14 @@ private FlightInfo executeQueryStatement(String peerIdentity, ConnectContext con
284284
}
285285
}
286286
} catch (Throwable e) {
287+
// GetFlightInfo failed (e.g. the BE Arrow schema fetch above timed out or returned an
288+
// error) after this query's coordinator may already have been deferred during planning.
289+
// No FlightInfo is returned, so no DoGet will ever pull this query's results; finalize
290+
// the deferred coordinator now (releasing its external-table batch SplitSource, query
291+
// queue slot and query registration) instead of leaking it until the next query starts
292+
// or the connection is torn down. The previous query's deferred coordinator was already
293+
// finalized at the top of this method, so this only closes this failed query. See #62259.
294+
connectContext.closeFlightSqlDeferredExecutors();
287295
String errMsg = "get flight info statement failed, " + e.getMessage() + ", " + Util.getRootCauseMessage(e)
288296
+ ", error code: " + connectContext.getState().getErrorCode() + ", error msg: "
289297
+ connectContext.getState().getErrorMessage();
Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.service.arrowflight;
19+
20+
import org.apache.doris.qe.ConnectContext;
21+
import org.apache.doris.qe.StmtExecutor;
22+
import org.apache.doris.service.arrowflight.results.FlightSqlChannel;
23+
import org.apache.doris.service.arrowflight.sessions.FlightSessionsManager;
24+
25+
import org.apache.arrow.flight.FlightDescriptor;
26+
import org.apache.arrow.flight.FlightProducer.CallContext;
27+
import org.apache.arrow.flight.Location;
28+
import org.apache.arrow.flight.sql.impl.FlightSql.CommandStatementQuery;
29+
import org.junit.Assert;
30+
import org.junit.Test;
31+
import org.mockito.MockedConstruction;
32+
import org.mockito.Mockito;
33+
34+
public class DorisFlightSqlProducerTest {
35+
36+
// Arrow Flight SQL keeps a query's coordinator alive across GetFlightInfo -> DoGet (see #62259):
37+
// executeAndSendResult() registers it as a deferred executor on the ConnectContext right after
38+
// submitting it to the BE. GetFlightInfo then still has to fetch the Arrow schema from the BE.
39+
// If that fetch fails (timeout / non-OK / empty / mismatched schema / RPC error), no FlightInfo
40+
// is returned, so no DoGet will ever pull this query's results. The deferred coordinator must be
41+
// finalized on this error path; otherwise its external-table batch SplitSource, query queue slot
42+
// and query registration leak until the next query starts or the connection is torn down.
43+
@Test
44+
public void testGetFlightInfoFinalizesDeferredExecutorWhenSchemaFetchFails() throws Exception {
45+
// A flight ConnectContext whose getFlightSqlChannel() works (the base context throws).
46+
ConnectContext ctx = Mockito.spy(new ConnectContext());
47+
Mockito.doReturn(Mockito.mock(FlightSqlChannel.class)).when(ctx).getFlightSqlChannel();
48+
49+
// Stands in for the just-planned external-table query whose results DoGet would pull from BE.
50+
StmtExecutor deferred = Mockito.mock(StmtExecutor.class);
51+
52+
FlightSessionsManager sessionsManager = Mockito.mock(FlightSessionsManager.class);
53+
Mockito.when(sessionsManager.getConnectContext(Mockito.anyString())).thenReturn(ctx);
54+
55+
CallContext callContext = Mockito.mock(CallContext.class);
56+
Mockito.when(callContext.peerIdentity()).thenReturn("token");
57+
58+
DorisFlightSqlProducer producer = new DorisFlightSqlProducer(
59+
Location.forGrpcInsecure("127.0.0.1", 9090), sessionsManager);
60+
try (MockedConstruction<FlightSqlConnectProcessor> mocked = Mockito.mockConstruction(
61+
FlightSqlConnectProcessor.class, (mock, context) -> {
62+
// handleQuery plans + submits to BE and defers the coordinator (coordBase == coord),
63+
// exactly as executeAndSendResult() does for an Arrow Flight external-table scan.
64+
Mockito.doAnswer(invocation -> {
65+
ctx.setReturnResultFromLocal(false);
66+
ctx.addFlightSqlDeferredExecutor(deferred);
67+
return null;
68+
}).when(mock).handleQuery(Mockito.anyString());
69+
// The Arrow schema fetch fails after the coordinator was already deferred.
70+
Mockito.doThrow(new RuntimeException("fetch arrow flight schema timeout"))
71+
.when(mock).fetchArrowFlightSchema(Mockito.anyInt());
72+
})) {
73+
CommandStatementQuery request = CommandStatementQuery.newBuilder().setQuery("select 1").build();
74+
FlightDescriptor descriptor = FlightDescriptor.command(new byte[0]);
75+
76+
try {
77+
producer.getFlightInfoStatement(request, callContext, descriptor);
78+
Assert.fail("expected the schema fetch failure to propagate as a CallStatus");
79+
} catch (Throwable expected) {
80+
// GetFlightInfo is expected to fail; the point of the test is what happens to the
81+
// deferred coordinator, not the thrown status itself.
82+
}
83+
84+
// The deferred coordinator of the failed query is finalized on the error path instead of
85+
// leaking until the next query / connection teardown.
86+
Mockito.verify(deferred).finalizeArrowFlightQuery();
87+
88+
// It is also removed from the deferred list, so a later teardown does not finalize it
89+
// again (no double-close, no retained reference).
90+
ctx.closeFlightSqlDeferredExecutors();
91+
Mockito.verify(deferred, Mockito.times(1)).finalizeArrowFlightQuery();
92+
} finally {
93+
producer.close();
94+
}
95+
}
96+
}

0 commit comments

Comments
 (0)