Skip to content

Commit 5ce3096

Browse files
GJ100柳吟风morningman
authored andcommitted
[fix](arrow-flight-sql) Close temporary VectorSchemaRoot in createPreparedStatement to fix FE direct memory leak (#65311)
``` ### What problem does this PR solve? Issue Number: close #65305 Problem Summary: `DorisFlightSqlProducer#createPreparedStatement` creates two `VectorSchemaRoot` instances (an empty one for the parameter schema, and one from `FlightSqlChannel#createOneOneSchemaRoot(...)` for the result metadata), extracts only their `Schema`, and never closes them. `createOneOneSchemaRoot` allocates a `VarCharVector` from the channel `RootAllocator` (off-heap, Netty pooled direct buffer). Since the returned `VectorSchemaRoot` is never closed, its off-heap buffer is leaked on every prepare call (effectively every Arrow Flight query, because ADBC prepares each statement). The buffer cannot be reclaimed by GC (strongly referenced by the allocator) nor by closing the client session (`FlightSqlChannel#close()` only invalidates the result cache and does not close the allocator; the leaked root is never stored in any session map). Under continuous Arrow Flight query load this makes FE direct memory grow monotonically until: `java.lang.OutOfMemoryError: Cannot reserve ... bytes of direct buffer memory (Internal; Prepare)` This PR wraps both temporary roots in try-with-resources so their off-heap buffers are released after the `Schema` is extracted. `Schema` is an immutable POJO and remains valid after the root is closed. Compare with `getCatalogs`/`getSchemas`/`getTables` in the same producer, which already use try-with-resources. --------- Signed-off-by: 柳吟风 <[email protected]> Signed-off-by: morningman <[email protected]> Co-authored-by: 柳吟风 <[email protected]> Co-authored-by: morningman <[email protected]>
1 parent 061738e commit 5ce3096

3 files changed

Lines changed: 163 additions & 4 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducer.java‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -356,14 +356,22 @@ public void createPreparedStatement(final ActionCreatePreparedStatementRequest r
356356
final ByteString handle = ByteString.copyFromUtf8(context.peerIdentity() + ":" + preparedStatementId);
357357
connectContext.addPreparedQuery(preparedStatementId, query);
358358

359-
VectorSchemaRoot emptyVectorSchemaRoot = new VectorSchemaRoot(new ArrayList<>(), new ArrayList<>());
360-
final Schema parameterSchema = emptyVectorSchemaRoot.getSchema();
359+
// Close the temporary VectorSchemaRoot after extracting its Schema, otherwise the
360+
// off-heap buffers backing its vectors are leaked on every prepare (FE direct memory leak).
361+
final Schema parameterSchema;
362+
try (VectorSchemaRoot emptyVectorSchemaRoot =
363+
new VectorSchemaRoot(new ArrayList<>(), new ArrayList<>())) {
364+
parameterSchema = emptyVectorSchemaRoot.getSchema();
365+
}
361366
// TODO FE does not have the ability to convert root fragment output expr into arrow schema.
362367
// However, the metaData schema returned by createPreparedStatement is usually not used by the client,
363368
// but it cannot be empty, otherwise it will be mistaken by the client as an updata statement.
364369
// see: https://github.com/apache/arrow/issues/38911
365-
Schema metaData = connectContext.getFlightSqlChannel()
366-
.createOneOneSchemaRoot("ResultMeta", "UNIMPLEMENTED").getSchema();
370+
final Schema metaData;
371+
try (VectorSchemaRoot metaSchemaRoot = connectContext.getFlightSqlChannel()
372+
.createOneOneSchemaRoot("ResultMeta", "UNIMPLEMENTED")) {
373+
metaData = metaSchemaRoot.getSchema();
374+
}
367375
listener.onNext(new Result(
368376
Any.pack(buildCreatePreparedStatementResult(handle, parameterSchema, metaData)).toByteArray()));
369377
} catch (Exception e) {

‎fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/results/FlightSqlChannel.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -156,6 +156,12 @@ public long resultNum() {
156156
return resultCache.size();
157157
}
158158

159+
// Returns the off-heap memory (bytes) currently held by this channel's Arrow allocator.
160+
// Exposed so tests can assert prepared-statement handling does not leak VectorSchemaRoot buffers.
161+
public long getAllocatedMemory() {
162+
return allocator.getAllocatedMemory();
163+
}
164+
159165
public void reset() {
160166
resultCache.invalidateAll();
161167
}
Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.service.arrowflight;
19+
20+
import org.apache.doris.common.FeConstants;
21+
import org.apache.doris.qe.ConnectContext;
22+
import org.apache.doris.service.arrowflight.results.FlightSqlChannel;
23+
import org.apache.doris.service.arrowflight.sessions.FlightSessionsManager;
24+
import org.apache.doris.service.arrowflight.sessions.FlightSqlConnectContext;
25+
26+
import org.apache.arrow.flight.FlightProducer.CallContext;
27+
import org.apache.arrow.flight.FlightProducer.StreamListener;
28+
import org.apache.arrow.flight.Location;
29+
import org.apache.arrow.flight.Result;
30+
import org.apache.arrow.flight.sql.impl.FlightSql.ActionCreatePreparedStatementRequest;
31+
import org.junit.After;
32+
import org.junit.Assert;
33+
import org.junit.Before;
34+
import org.junit.Test;
35+
import org.mockito.Mockito;
36+
37+
import java.util.concurrent.CountDownLatch;
38+
import java.util.concurrent.TimeUnit;
39+
import java.util.concurrent.atomic.AtomicInteger;
40+
41+
public class DorisFlightSqlProducerTest {
42+
43+
private boolean prevRunningUnitTest;
44+
45+
@Before
46+
public void setUp() {
47+
// FlightSqlConnectContext.init() only reaches Env when this is false; keep it true so the
48+
// context can be built without a running FE.
49+
prevRunningUnitTest = FeConstants.runningUnitTest;
50+
FeConstants.runningUnitTest = true;
51+
}
52+
53+
@After
54+
public void tearDown() {
55+
FeConstants.runningUnitTest = prevRunningUnitTest;
56+
}
57+
58+
/**
59+
* Regression test for the FE direct-memory leak in {@code createPreparedStatement}
60+
* (issue apache/doris#65305, fixed by PR #65311).
61+
*
62+
* <p>Before the fix, each prepare allocated a {@link org.apache.arrow.vector.VectorSchemaRoot}
63+
* via {@code FlightSqlChannel.createOneOneSchemaRoot("ResultMeta", ...)} and read only its
64+
* {@code Schema}, never closing the root, leaking one off-heap {@code VarCharVector} buffer per
65+
* prepare. Because Arrow Flight has no parameter binding, every client query triggers a fresh
66+
* prepare, so the leak grows monotonically until {@code MaxDirectMemorySize} is exhausted.
67+
*
68+
* <p>This test drives {@code createPreparedStatement} against a real {@link FlightSqlChannel} many
69+
* times and asserts the channel's Arrow allocator holds zero bytes afterwards. It goes red if the
70+
* try-with-resources that closes the temporary roots is removed.
71+
*/
72+
@Test
73+
public void createPreparedStatementDoesNotLeakChannelAllocator() throws Exception {
74+
// A real flight session context owns a real FlightSqlChannel (and thus a real Arrow allocator),
75+
// so allocator bookkeeping is exercised for real instead of mocked away.
76+
FlightSqlConnectContext connectContext = new FlightSqlConnectContext("test-peer-identity");
77+
FlightSqlChannel channel = connectContext.getFlightSqlChannel();
78+
Assert.assertEquals("channel allocator should start empty", 0L, channel.getAllocatedMemory());
79+
80+
FlightSessionsManager sessionsManager = new FlightSessionsManager() {
81+
@Override
82+
public ConnectContext getConnectContext(String peerIdentity) {
83+
return connectContext;
84+
}
85+
86+
@Override
87+
public ConnectContext createConnectContext(String peerIdentity) {
88+
return connectContext;
89+
}
90+
91+
@Override
92+
public void closeConnectContext(String peerIdentity) {
93+
// not exercised by this test
94+
}
95+
};
96+
DorisFlightSqlProducer producer =
97+
new DorisFlightSqlProducer(Location.forGrpcInsecure("127.0.0.1", 9090), sessionsManager);
98+
99+
CallContext callContext = Mockito.mock(CallContext.class);
100+
Mockito.when(callContext.peerIdentity()).thenReturn("test-peer-identity");
101+
102+
final int rounds = 100;
103+
AtomicInteger errors = new AtomicInteger(0);
104+
try {
105+
// createPreparedStatement runs asynchronously and mutates a shared, non-thread-safe
106+
// ConnectContext, so drive it serially: each prepare completes before the next is issued.
107+
// The leak, if any, still accumulates on the single channel allocator across rounds.
108+
for (int i = 0; i < rounds; i++) {
109+
CountDownLatch finished = new CountDownLatch(1);
110+
StreamListener<Result> listener = new StreamListener<Result>() {
111+
@Override
112+
public void onNext(Result val) {
113+
// discard the placeholder prepared-statement result
114+
}
115+
116+
@Override
117+
public void onError(Throwable t) {
118+
errors.incrementAndGet();
119+
finished.countDown();
120+
}
121+
122+
@Override
123+
public void onCompleted() {
124+
finished.countDown();
125+
}
126+
};
127+
ActionCreatePreparedStatementRequest request = ActionCreatePreparedStatementRequest.newBuilder()
128+
.setQuery("select * from t where id = " + i).build();
129+
producer.createPreparedStatement(request, callContext, listener);
130+
Assert.assertTrue("createPreparedStatement #" + i + " did not finish in time",
131+
finished.await(30, TimeUnit.SECONDS));
132+
}
133+
134+
// Guard against a false pass: if a prepare failed before reaching the allocation, no buffer
135+
// would be leaked and the memory assertion below could not detect a regression.
136+
Assert.assertEquals("no createPreparedStatement call should fail", 0, errors.get());
137+
// Every temporary VectorSchemaRoot must have been closed, so the channel's Arrow allocator
138+
// is back to zero. Reverting the fix leaves `rounds` ResultMeta buffers allocated here.
139+
Assert.assertEquals("createPreparedStatement leaked off-heap memory in the channel allocator",
140+
0L, channel.getAllocatedMemory());
141+
} finally {
142+
producer.close();
143+
}
144+
}
145+
}

0 commit comments

Comments
 (0)