Skip to content

Commit 826d25d

Browse files
committed
[fix](arrow-flight) Finalize deferred coordinator when GetFlightInfo fails after deferral
For an Arrow Flight external-table scan, executeAndSendResult() registers the query's coordinator as a deferred executor on the ConnectContext (kept alive across GetFlightInfo -> DoGet, see apache#62259) right after submitting it to the BE. GetFlightInfo then still has to fetch the Arrow schema from the BE. If that fetch fails (timeout / non-OK / empty / mismatched schema / RPC error), FlightSqlConnectProcessor.close() skips the deferred executor and the outer catch only rethrows. No FlightInfo is returned, so no DoGet will ever pull the results, yet the coordinator (its external-table batch SplitSource, the query queue slot and the query registration) stays alive until the next query starts or the connection is torn down. Finalize the deferred coordinator on the GetFlightInfo error path: executeQueryStatement's catch now calls closeFlightSqlDeferredExecutors(). At that point the list holds only this failed query's coordinator (the previous one was already finalized at the top of the method), and this covers every post-deferral failure, not just the schema fetch. Add DorisFlightSqlProducerTest.testGetFlightInfoFinalizesDeferredExecutorWhen SchemaFetchFails, which defers a coordinator then fails the schema fetch and asserts the deferred executor is finalized (verified to fail without the fix).
1 parent d4a974d commit 826d25d

2 files changed

Lines changed: 73 additions & 0 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducer.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -284,6 +284,14 @@ private FlightInfo executeQueryStatement(String peerIdentity, ConnectContext con
284284
}
285285
}
286286
} catch (Throwable e) {
287+
// GetFlightInfo failed (e.g. the BE Arrow schema fetch above timed out or returned an
288+
// error) after this query's coordinator may already have been deferred during planning.
289+
// No FlightInfo is returned, so no DoGet will ever pull this query's results; finalize
290+
// the deferred coordinator now (releasing its external-table batch SplitSource, query
291+
// queue slot and query registration) instead of leaking it until the next query starts
292+
// or the connection is torn down. The previous query's deferred coordinator was already
293+
// finalized at the top of this method, so this only closes this failed query. See #62259.
294+
connectContext.closeFlightSqlDeferredExecutors();
287295
String errMsg = "get flight info statement failed, " + e.getMessage() + ", " + Util.getRootCauseMessage(e)
288296
+ ", error code: " + connectContext.getState().getErrorCode() + ", error msg: "
289297
+ connectContext.getState().getErrorMessage();

‎fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducerTest.java‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,19 +19,23 @@
1919

2020
import org.apache.doris.common.FeConstants;
2121
import org.apache.doris.qe.ConnectContext;
22+
import org.apache.doris.qe.StmtExecutor;
2223
import org.apache.doris.service.arrowflight.results.FlightSqlChannel;
2324
import org.apache.doris.service.arrowflight.sessions.FlightSessionsManager;
2425
import org.apache.doris.service.arrowflight.sessions.FlightSqlConnectContext;
2526

27+
import org.apache.arrow.flight.FlightDescriptor;
2628
import org.apache.arrow.flight.FlightProducer.CallContext;
2729
import org.apache.arrow.flight.FlightProducer.StreamListener;
2830
import org.apache.arrow.flight.Location;
2931
import org.apache.arrow.flight.Result;
3032
import org.apache.arrow.flight.sql.impl.FlightSql.ActionCreatePreparedStatementRequest;
33+
import org.apache.arrow.flight.sql.impl.FlightSql.CommandStatementQuery;
3134
import org.junit.After;
3235
import org.junit.Assert;
3336
import org.junit.Before;
3437
import org.junit.Test;
38+
import org.mockito.MockedConstruction;
3539
import org.mockito.Mockito;
3640

3741
import java.util.concurrent.CountDownLatch;
@@ -142,4 +146,65 @@ public void onCompleted() {
142146
producer.close();
143147
}
144148
}
149+
150+
// Arrow Flight SQL keeps a query's coordinator alive across GetFlightInfo -> DoGet (see #62259):
151+
// executeAndSendResult() registers it as a deferred executor on the ConnectContext right after
152+
// submitting it to the BE. GetFlightInfo then still has to fetch the Arrow schema from the BE.
153+
// If that fetch fails (timeout / non-OK / empty / mismatched schema / RPC error), no FlightInfo
154+
// is returned, so no DoGet will ever pull this query's results. The deferred coordinator must be
155+
// finalized on this error path; otherwise its external-table batch SplitSource, query queue slot
156+
// and query registration leak until the next query starts or the connection is torn down.
157+
@Test
158+
public void testGetFlightInfoFinalizesDeferredExecutorWhenSchemaFetchFails() throws Exception {
159+
// A flight ConnectContext whose getFlightSqlChannel() works (the base context throws).
160+
ConnectContext ctx = Mockito.spy(new ConnectContext());
161+
Mockito.doReturn(Mockito.mock(FlightSqlChannel.class)).when(ctx).getFlightSqlChannel();
162+
163+
// Stands in for the just-planned external-table query whose results DoGet would pull from BE.
164+
StmtExecutor deferred = Mockito.mock(StmtExecutor.class);
165+
166+
FlightSessionsManager sessionsManager = Mockito.mock(FlightSessionsManager.class);
167+
Mockito.when(sessionsManager.getConnectContext(Mockito.anyString())).thenReturn(ctx);
168+
169+
CallContext callContext = Mockito.mock(CallContext.class);
170+
Mockito.when(callContext.peerIdentity()).thenReturn("token");
171+
172+
DorisFlightSqlProducer producer = new DorisFlightSqlProducer(
173+
Location.forGrpcInsecure("127.0.0.1", 9090), sessionsManager);
174+
try (MockedConstruction<FlightSqlConnectProcessor> mocked = Mockito.mockConstruction(
175+
FlightSqlConnectProcessor.class, (mock, context) -> {
176+
// handleQuery plans + submits to BE and defers the coordinator (coordBase == coord),
177+
// exactly as executeAndSendResult() does for an Arrow Flight external-table scan.
178+
Mockito.doAnswer(invocation -> {
179+
ctx.setReturnResultFromLocal(false);
180+
ctx.addFlightSqlDeferredExecutor(deferred);
181+
return null;
182+
}).when(mock).handleQuery(Mockito.anyString());
183+
// The Arrow schema fetch fails after the coordinator was already deferred.
184+
Mockito.doThrow(new RuntimeException("fetch arrow flight schema timeout"))
185+
.when(mock).fetchArrowFlightSchema(Mockito.anyInt());
186+
})) {
187+
CommandStatementQuery request = CommandStatementQuery.newBuilder().setQuery("select 1").build();
188+
FlightDescriptor descriptor = FlightDescriptor.command(new byte[0]);
189+
190+
try {
191+
producer.getFlightInfoStatement(request, callContext, descriptor);
192+
Assert.fail("expected the schema fetch failure to propagate as a CallStatus");
193+
} catch (Throwable expected) {
194+
// GetFlightInfo is expected to fail; the point of the test is what happens to the
195+
// deferred coordinator, not the thrown status itself.
196+
}
197+
198+
// The deferred coordinator of the failed query is finalized on the error path instead of
199+
// leaking until the next query / connection teardown.
200+
Mockito.verify(deferred).finalizeArrowFlightQuery();
201+
202+
// It is also removed from the deferred list, so a later teardown does not finalize it
203+
// again (no double-close, no retained reference).
204+
ctx.closeFlightSqlDeferredExecutors();
205+
Mockito.verify(deferred, Mockito.times(1)).finalizeArrowFlightQuery();
206+
} finally {
207+
producer.close();
208+
}
209+
}
145210
}

0 commit comments

Comments
 (0)