Skip to content

Commit c2cb912

Browse files
authored
[bugfix](arrowflight) fix some off heap memory leak on FE (#66437)
### What problem does this PR solve? Issue Number: #65615
1 parent d7e76f7 commit c2cb912

6 files changed

Lines changed: 30 additions & 10 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlConnectProcessor.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -146,11 +146,11 @@ public void fetchArrowFlightSchema(int timeoutMs) {
146146
}
147147
endpointLoc.setResultPublicAccessAddr(resultPublicAccessAddr);
148148
if (pResult.hasSchema() && pResult.getSchema().size() > 0) {
149-
RootAllocator rootAllocator = new RootAllocator(Integer.MAX_VALUE);
150-
ArrowStreamReader arrowStreamReader = new ArrowStreamReader(
151-
new ByteArrayInputStream(pResult.getSchema().toByteArray()), rootAllocator);
152-
try {
149+
try (RootAllocator rootAllocator = new RootAllocator(Integer.MAX_VALUE);
150+
ArrowStreamReader arrowStreamReader = new ArrowStreamReader(
151+
new ByteArrayInputStream(pResult.getSchema().toByteArray()), rootAllocator)) {
153152
Schema schema;
153+
// SchemaRoot belongs to ArrowStreamReader, it will be released when ArrowStreamReader is closed
154154
VectorSchemaRoot root = arrowStreamReader.getVectorSchemaRoot();
155155
List<FieldVector> fieldVectors = root.getFieldVectors();
156156
if (fieldVectors.size() != resultOutputExprs.size()) {

‎fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/results/FlightSqlChannel.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -168,6 +168,7 @@ public void reset() {
168168

169169
public void close() {
170170
reset();
171+
allocator.close();
171172
}
172173

173174
private static class ResultRemovalListener implements RemovalListener<String, FlightSqlResultCacheEntry> {

‎fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/results/FlightSqlResultCacheEntry.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ public String getQuery() {
4242

4343
@Override
4444
public void close() throws Exception {
45-
vectorSchemaRoot.clear();
45+
vectorSchemaRoot.close();
4646
}
4747

4848
@Override

‎fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/sessions/FlightSqlConnectContext.java‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -62,9 +62,6 @@ public String getClientIP() {
6262

6363
@Override
6464
protected void closeChannel() {
65-
if (flightSqlChannel != null) {
66-
flightSqlChannel.close();
67-
}
6865
connectScheduler.getFlightSqlConnectPoolMgr().unregisterConnection(this);
6966
}
7067

‎fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/sessions/FlightSqlConnectPoolMgr.java‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import org.apache.doris.qe.ConnectContext;
2121
import org.apache.doris.qe.ConnectContext.ConnectType;
2222
import org.apache.doris.qe.ConnectPoolMgr;
23+
import org.apache.doris.service.arrowflight.results.FlightSqlChannel;
2324

2425
import com.google.common.collect.Maps;
2526
import org.apache.logging.log4j.LogManager;
@@ -55,10 +56,24 @@ public int registerConnection(ConnectContext ctx) {
5556

5657
@Override
5758
public void unregisterConnection(ConnectContext ctx) {
59+
// All Flight SQL session teardown paths (idle/query timeout, bearer token expiry, and
60+
// explicit CloseSession) reach here. Release channel-cached Arrow results before removing
61+
// the context from the pool.
62+
FlightSqlChannel flightSqlChannel = ctx.getFlightSqlChannel();
63+
if (flightSqlChannel != null) {
64+
try {
65+
flightSqlChannel.close();
66+
} catch (Throwable t) {
67+
// RootAllocator.close() marks the allocator closed before it reports outstanding
68+
// bytes. The error is actionable, but session teardown must still release the
69+
// coordinator, transaction and pool/token bookkeeping below.
70+
LOG.warn("failed to close Flight SQL channel while unregistering connection {}, peer identity {}",
71+
ctx.getConnectionId(), ctx.getPeerIdentity(), t);
72+
}
73+
}
5874
// Finalize any Arrow Flight query whose coordinator was kept alive across the
5975
// GetFlightInfo -> DoGet phases (see #62259), releasing its resources (e.g. external-table
60-
// batch SplitSources and the query queue slot) when the connection is torn down: idle/query
61-
// timeout, bearer token expiry, or explicit CloseSession all reach here.
76+
// batch SplitSources and the query queue slot).
6277
ctx.closeFlightSqlDeferredExecutors();
6378
ctx.closeTxn();
6479
if (connectionMap.remove(ctx.getConnectionId()) != null) {

‎fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/sessions/FlightSqlConnectPoolMgrTest.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
package org.apache.doris.service.arrowflight.sessions;
1919

2020
import org.apache.doris.qe.ConnectContext;
21+
import org.apache.doris.service.arrowflight.results.FlightSqlChannel;
2122

2223
import org.junit.Assert;
2324
import org.junit.Test;
@@ -34,11 +35,14 @@ public class FlightSqlConnectPoolMgrTest {
3435
public void testUnregisterConnectionFinalizesDeferredExecutors() {
3536
FlightSqlConnectPoolMgr poolMgr = new FlightSqlConnectPoolMgr(100);
3637
ConnectContext ctx = Mockito.mock(ConnectContext.class);
38+
FlightSqlChannel channel = Mockito.mock(FlightSqlChannel.class);
39+
Mockito.when(ctx.getFlightSqlChannel()).thenReturn(channel);
3740

3841
poolMgr.unregisterConnection(ctx);
3942

4043
// The deferred coordinators must be released on teardown even though this connection was
4144
// never registered in the pool (an abandoned connection is still cleaned up, not leaked).
45+
Mockito.verify(channel).close();
4246
Mockito.verify(ctx).closeFlightSqlDeferredExecutors();
4347
}
4448

@@ -49,13 +53,16 @@ public void testUnregisterConnectionFinalizesDeferredExecutors() {
4953
public void testUnregisterRegisteredConnectionFinalizesDeferredExecutors() {
5054
FlightSqlConnectPoolMgr poolMgr = new FlightSqlConnectPoolMgr(100);
5155
ConnectContext ctx = Mockito.mock(ConnectContext.class);
56+
FlightSqlChannel channel = Mockito.mock(FlightSqlChannel.class);
57+
Mockito.when(ctx.getFlightSqlChannel()).thenReturn(channel);
5258
Mockito.when(ctx.getConnectionId()).thenReturn(7);
5359
Mockito.when(ctx.getConnectType()).thenReturn(ConnectContext.ConnectType.ARROW_FLIGHT_SQL);
5460
Mockito.when(ctx.getPeerIdentity()).thenReturn("token-7");
5561
poolMgr.getConnectionMap().put(7, ctx);
5662

5763
poolMgr.unregisterConnection(ctx);
5864

65+
Mockito.verify(channel).close();
5966
Mockito.verify(ctx).closeFlightSqlDeferredExecutors();
6067
Assert.assertNull(poolMgr.getConnectionMap().get(7));
6168
}

0 commit comments

Comments
 (0)