Skip to content

Commit b446ac8

Browse files
morningmanclaude
andcommitted
[fix](arrow-flight) Make shared-pool admission atomic and harden the quota suite
Addresses the static review's findings and the pipeline failure the earlier fixes exposed. Production: - registerConnection checks and reserves the pool, Flight sub-quota and per-user quotas in one critical section (admissionLock); the earlier per-counter increment-check-rollback let a doomed attempt hold the last pool slot across a concurrent attempt's checks and refuse a connection that fits in every serial ordering. - getConnectContext serializes session creation and re-checks the peer-identity index, so exactly one ConnectContext is published per bearer token; and after registration it re-validates the token under the same lock and unregisters the session if the token was invalidated meanwhile (concurrent CloseSession, LRU eviction), so no session lingers in the pool without its token. - The bearer token cache size is computed by effectiveTokenCacheSize, which floors only the effective sub-quota at 1 (a legal sub-quota of 0 at qe_max_connection=1 still needs one token so the first request reaches the pool and is refused with RESOURCE_EXHAUSTED, not UNAUTHENTICATED). An illegal arrow_flight_token_cache_size (<= 0) is left as-is, keeping the base's loud failure. Its diagnostic reflects the floor. Tests: - DorisFlightSqlServiceTest: effectiveTokenCacheSize floors the sub-quota but not the cap. - ConnectPoolMgrTest: a deterministic guard that freezes one registration inside getMaxConn while a connection that fits races it, and a zero-sub-quota boundary test. The guard fails without the lock. - ConnectionExceedTest: concurrent first requests on one bearer token publish exactly one session (each worker installs its own thread-local static mocks over the one shared scheduler). - FlightProtocolAdapterTest: drop the allocator-close-failure test; forcing the failure needed a retained Arrow buffer that could not be released afterward (the allocator is already closed), leaking native memory. The normal release path stays covered. - test_connection_quota: the quota is enforced when the session is registered, before the probe query runs, so only RESOURCE_EXHAUSTED is a refusal; an INTERNAL after admission is logged and tolerated (the connectionsOf() checks confirm the session), and a reopened session's count is asserted before it is closed. This stops the suite from failing on an environment-specific probe-query error after admission, which is outside a connection-quota test's scope. Co-Authored-By: Claude Opus 4.8 <[email protected]>
1 parent b2c6de9 commit b446ac8

9 files changed

Lines changed: 287 additions & 66 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/arrowflight/DorisFlightSqlService.java‎

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import org.apache.doris.qe.ConnectPoolMgr;
2828
import org.apache.doris.service.FrontendOptions;
2929

30+
import com.google.common.annotations.VisibleForTesting;
3031
import io.grpc.ServerBuilder;
3132
import org.apache.arrow.flight.FlightServer;
3233
import org.apache.arrow.flight.Location;
@@ -49,6 +50,21 @@ public class DorisFlightSqlService {
4950
private final FlightSessionsManager flightSessionsManager;
5051
private volatile boolean running;
5152

53+
/**
54+
* The bearer token cache size: the effective Flight sub-quota (a session opens on a token, so the
55+
* sub-quota is what bounds live sessions), capped by {@code arrow_flight_token_cache_size}. The
56+
* sub-quota is floored at 1 -- a legal sub-quota of 0 ({@code qe_max_connection = 1}) still needs
57+
* one token so the first request reaches the pool and is refused with RESOURCE_EXHAUSTED, instead
58+
* of a {@code maximumSize(0)} cache evicting the freshly issued token and answering UNAUTHENTICATED.
59+
* The floor is NOT applied to {@code arrow_flight_token_cache_size}: an illegal value there
60+
* ({@literal <= 0}) stays the loud failure it is on the base (Guava rejects a negative maximumSize;
61+
* 0 evicts every token) rather than the FE silently running on a one-token cache.
62+
*/
63+
@VisibleForTesting
64+
static int effectiveTokenCacheSize(int flightMaxConnections, int tokenCacheConfig) {
65+
return Math.min(Math.max(1, flightMaxConnections), tokenCacheConfig);
66+
}
67+
5268
public DorisFlightSqlService(int port) {
5369
BufferAllocator allocator = new RootAllocator();
5470
// arrow flight sql is a stateless protocol, connection is usually not actively disconnected.
@@ -66,10 +82,7 @@ public DorisFlightSqlService(int port) {
6682
+ " the default before the pools were merged) or set it below qe_max_connection",
6783
Config.arrow_flight_max_connections, Config.qe_max_connection, flightMaxConnections);
6884
}
69-
// The token cache holds as many tokens as the sub-quota allows, capped by
70-
// arrow_flight_token_cache_size: a session opens on a token, so this is what bounds Flight
71-
// sessions in practice (the oldest token, and its session, is evicted first).
72-
int tokenCacheSize = Math.min(flightMaxConnections, Config.arrow_flight_token_cache_size);
85+
int tokenCacheSize = effectiveTokenCacheSize(flightMaxConnections, Config.arrow_flight_token_cache_size);
7386
this.flightTokenManager = new FlightTokenManagerImpl(tokenCacheSize,
7487
Config.arrow_flight_token_alive_time_second);
7588
this.flightSessionsManager = new FlightSessionsWithTokenManager(flightTokenManager);

‎fe/fe-core/src/main/java/org/apache/doris/arrowflight/sessions/FlightSessionsWithTokenManager.java‎

Lines changed: 28 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,8 @@ public class FlightSessionsWithTokenManager implements FlightSessionsManager {
3636
private static final Logger LOG = LogManager.getLogger(FlightSessionsWithTokenManager.class);
3737

3838
private final FlightTokenManager flightTokenManager;
39+
// Serializes session creation so exactly one ConnectContext is published per bearer token.
40+
private final Object sessionCreationLock = new Object();
3941

4042
public FlightSessionsWithTokenManager(FlightTokenManager flightTokenManager) {
4143
this.flightTokenManager = flightTokenManager;
@@ -46,11 +48,22 @@ public ConnectContext getConnectContext(String peerIdentity) {
4648
try {
4749
ConnectContext connectContext = ExecuteEnv.getInstance().getScheduler()
4850
.getContextWithPeerIdentity(peerIdentity);
49-
if (null == connectContext) {
50-
connectContext = createConnectContext(peerIdentity);
51+
if (null != connectContext) {
5152
return connectContext;
5253
}
53-
return connectContext;
54+
// Publish exactly one context per bearer token. Two concurrent first requests would both
55+
// find none above and, without this, both build and register one: two pool slots and quota
56+
// counters for one token, with the peer-identity index naming only the last, so evicting
57+
// that one leaves the other orphaned in the pool until wait_timeout. Serialize creation and
58+
// re-check the index inside, so the loser returns the winner's published context.
59+
synchronized (sessionCreationLock) {
60+
connectContext = ExecuteEnv.getInstance().getScheduler()
61+
.getContextWithPeerIdentity(peerIdentity);
62+
if (null != connectContext) {
63+
return connectContext;
64+
}
65+
return createConnectContext(peerIdentity);
66+
}
5467
} catch (FlightRuntimeException e) {
5568
// Already the status the client is meant to see (a connection refused for its limit).
5669
throw e;
@@ -93,6 +106,18 @@ public ConnectContext createConnectContext(String peerIdentity) {
93106
TokenMasker.tokenId(peerIdentity), connectContext.getQualifiedUser(), errMsg);
94107
throw CallStatus.RESOURCE_EXHAUSTED.withDescription(errMsg).toRuntimeException();
95108
}
109+
// Called under sessionCreationLock (see getConnectContext). The token can be invalidated between
110+
// the validateToken above and publishing the session here -- a concurrent CloseSession, or the
111+
// same-segment LRU evicting it -- which would leave a session in the pool whose token no longer
112+
// exists, holding its pool/user/Flight slots until wait_timeout. Re-check the token and, if it is
113+
// gone, unregister the session so no orphan lingers. (The token lifecycle moves under one owner
114+
// in the next PR, which closes the residual window after this re-check.)
115+
try {
116+
flightTokenManager.validateToken(peerIdentity);
117+
} catch (IllegalArgumentException e) {
118+
connectScheduler.getConnectPoolMgr().unregisterConnection(connectContext);
119+
throw e;
120+
}
96121
return connectContext;
97122
}
98123

‎fe/fe-core/src/main/java/org/apache/doris/arrowflight/tokens/FlightTokenManagerImpl.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -160,11 +160,11 @@ public FlightTokenDetails validateToken(final String token) throws IllegalArgume
160160
throw new IllegalArgumentException("invalid bearer token, token id: " + TokenMasker.tokenId(token)
161161
+ ", try reconnect, bearer token may not be created, or may have been evict, search for this "
162162
+ "token id in fe.log to see the evict reason. the token cache is sized to " + this.cacheSize
163-
+ " tokens = min(effective Arrow Flight SQL sub-quota of the connection pool "
163+
+ " tokens = min(max(1, effective Arrow Flight SQL sub-quota of the connection pool "
164164
+ ConnectPoolMgr.effectiveFlightMaxConnections(Config.qe_max_connection,
165165
Config.arrow_flight_max_connections)
166166
+ " [fe.conf `arrow_flight_max_connections`=" + Config.arrow_flight_max_connections
167-
+ " within `qe_max_connection`=" + Config.qe_max_connection + "], `arrow_flight_token_cache_size`="
167+
+ " within `qe_max_connection`=" + Config.qe_max_connection + "]), `arrow_flight_token_cache_size`="
168168
+ Config.arrow_flight_token_cache_size + "), `arrow_flight_token_alive_time_second`="
169169
+ this.cacheExpiration);
170170
}

‎fe/fe-core/src/main/java/org/apache/doris/qe/ConnectPoolMgr.java‎

Lines changed: 29 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,9 @@ public class ConnectPoolMgr {
6060
protected final Map<String, AtomicInteger> connByUser = Maps.newConcurrentMap();
6161
// Arrow Flight SQL: peer identity (bearer token) -> connection id
6262
private final Map<String, Integer> peerIdentity2ConnectionId = Maps.newConcurrentMap();
63+
// Serializes the check-and-reserve of the pool/Flight/user quotas in registerConnection, so a
64+
// doomed attempt cannot hold one counter's reservation across another attempt's checks.
65+
private final Object admissionLock = new Object();
6366

6467
// valid trace id -> query id
6568
protected final Map<String, TUniqueId> traceId2QueryId = Maps.newConcurrentMap();
@@ -110,29 +113,34 @@ public void timeoutChecker(long now) {
110113
// Return >=0 means register failed, and return value is current connection num.
111114
public int registerConnection(ConnectContext ctx) {
112115
boolean flight = isFlight(ctx);
113-
if (numberConnection.incrementAndGet() > maxConnections) {
114-
numberConnection.decrementAndGet();
115-
return numberConnection.get();
116-
}
117-
if (flight && numberFlightConnection.incrementAndGet() > flightMaxConnections) {
118-
numberFlightConnection.decrementAndGet();
119-
numberConnection.decrementAndGet();
120-
return numberConnection.get();
121-
}
122-
// Check user
123-
connByUser.putIfAbsent(ctx.getQualifiedUser(), new AtomicInteger(0));
124-
AtomicInteger conns = connByUser.get(ctx.getQualifiedUser());
125-
if (conns.incrementAndGet() > ctx.getEnv().getAuth().getMaxConn(ctx.getQualifiedUser())) {
126-
conns.decrementAndGet();
116+
String qualifiedUser = ctx.getQualifiedUser();
117+
long userLimit = ctx.getEnv().getAuth().getMaxConn(qualifiedUser);
118+
// Check and reserve the pool, Flight sub-quota and per-user quotas as one transition. Doing it
119+
// under a lock (rather than the earlier increment-check-rollback per counter) keeps a doomed
120+
// attempt from reserving one counter across another attempt's checks: a second Flight attempt
121+
// that will fail its sub-quota could otherwise briefly hold the last pool slot and make a
122+
// concurrent MySQL connection - which fits in every serial ordering - be refused. Registration
123+
// is once per connection, not per query, so this is not a hot path.
124+
synchronized (admissionLock) {
125+
if (numberConnection.get() >= maxConnections) {
126+
return numberConnection.get();
127+
}
128+
if (flight && numberFlightConnection.get() >= flightMaxConnections) {
129+
return numberConnection.get();
130+
}
131+
AtomicInteger conns = connByUser.computeIfAbsent(qualifiedUser, u -> new AtomicInteger(0));
132+
if (conns.get() >= userLimit) {
133+
return numberConnection.get();
134+
}
135+
numberConnection.incrementAndGet();
127136
if (flight) {
128-
numberFlightConnection.decrementAndGet();
137+
numberFlightConnection.incrementAndGet();
138+
}
139+
conns.incrementAndGet();
140+
connectionMap.put(ctx.getConnectionId(), ctx);
141+
if (flight) {
142+
peerIdentity2ConnectionId.put(ctx.getPeerIdentity(), ctx.getConnectionId());
129143
}
130-
numberConnection.decrementAndGet();
131-
return numberConnection.get();
132-
}
133-
connectionMap.put(ctx.getConnectionId(), ctx);
134-
if (flight) {
135-
peerIdentity2ConnectionId.put(ctx.getPeerIdentity(), ctx.getConnectionId());
136144
}
137145
return -1;
138146
}
Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.arrowflight;
19+
20+
import org.junit.jupiter.api.Assertions;
21+
import org.junit.jupiter.api.Test;
22+
23+
public class DorisFlightSqlServiceTest {
24+
25+
// The token cache size floors the effective Flight sub-quota at 1, but never the configured cap:
26+
// a legal sub-quota of 0 (qe_max_connection = 1) keeps one token so the first request reaches the
27+
// pool and is refused with RESOURCE_EXHAUSTED, while an illegal arrow_flight_token_cache_size (<= 0)
28+
// is passed through so the base's loud failure (Guava rejects a negative maximumSize; 0 evicts every
29+
// token) is preserved rather than the FE silently running on a one-token cache.
30+
@Test
31+
public void testEffectiveTokenCacheSizeFloorsOnlyTheSubQuota() {
32+
// Legal sub-quota of 0 -> floored to 1.
33+
Assertions.assertEquals(1, DorisFlightSqlService.effectiveTokenCacheSize(0, 4096));
34+
// Normal: the smaller of the sub-quota and the cap.
35+
Assertions.assertEquals(512, DorisFlightSqlService.effectiveTokenCacheSize(512, 4096));
36+
Assertions.assertEquals(4096, DorisFlightSqlService.effectiveTokenCacheSize(8192, 4096));
37+
Assertions.assertEquals(10, DorisFlightSqlService.effectiveTokenCacheSize(512, 10));
38+
// An illegal cap is not floored: it stays <= 0 for the base's loud failure.
39+
Assertions.assertEquals(0, DorisFlightSqlService.effectiveTokenCacheSize(0, 0));
40+
Assertions.assertEquals(-1, DorisFlightSqlService.effectiveTokenCacheSize(512, -1));
41+
}
42+
}

‎fe/fe-core/src/test/java/org/apache/doris/arrowflight/protocol/FlightProtocolAdapterTest.java‎

Lines changed: 0 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,6 @@
4444
import com.google.common.collect.Lists;
4545
import org.apache.arrow.flight.FlightRuntimeException;
4646
import org.apache.arrow.flight.FlightStatusCode;
47-
import org.apache.arrow.memory.ArrowBuf;
4847
import org.apache.thrift.TException;
4948
import org.junit.jupiter.api.AfterEach;
5049
import org.junit.jupiter.api.Assertions;
@@ -190,34 +189,6 @@ public void testReleaseSessionClosesTheChannelAndTearsTheSessionDown() {
190189
Mockito.verify(deferred, Mockito.times(1)).finalizeArrowFlightQuery();
191190
}
192191

193-
// The channel close can fail - the allocator refuses to close over a buffer still referenced
194-
// elsewhere - and the session is torn down all the same: the deferred query is finalized and
195-
// the session takes no further command.
196-
@Test
197-
public void testReleaseSessionTearsTheSessionDownEvenWhenTheChannelCloseFails() {
198-
ConnectContext ctx = flightSession();
199-
FlightProtocolAdapter adapter = FlightProtocolAdapter.of(ctx);
200-
StmtExecutor deferred = Mockito.mock(StmtExecutor.class);
201-
ctx.addFlightSqlDeferredExecutor(deferred);
202-
adapter.getChannel().addOKResult("query-1", "SELECT 1");
203-
// A buffer of the cached result held by someone else, as a DoGet still streaming it would:
204-
// closing the result does not free it, and the allocator's close throws over it. The
205-
// reference is never given back - the failure is the point of the test.
206-
ArrowBuf held = adapter.getChannel().getResult("query-1").getVectorSchemaRoot().getVector(0)
207-
.getDataBuffer();
208-
held.getReferenceManager().retain();
209-
210-
Assertions.assertDoesNotThrow(ctx::releaseProtocolSession);
211-
212-
// The close did fail: the held buffer is still accounted for...
213-
Assertions.assertTrue(adapter.getChannel().getAllocatedMemory() > 0);
214-
// ...and the session is down regardless.
215-
Mockito.verify(deferred, Mockito.times(1)).finalizeArrowFlightQuery();
216-
Assertions.assertTrue(ctx.getFlightSqlDeferredExecutors().isEmpty());
217-
FlightRuntimeException closed = Assertions.assertThrows(FlightRuntimeException.class,
218-
() -> adapter.runCommand(ctx, () -> { }));
219-
Assertions.assertEquals(FlightStatusCode.UNAUTHENTICATED, closed.status().code());
220-
}
221192

222193
@Test
223194
public void testCommandsOfOneSessionRunOneAtATime() throws Exception {

‎fe/fe-core/src/test/java/org/apache/doris/mysql/ConnectionExceedTest.java‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
import org.apache.arrow.flight.FlightStatusCode;
3737
import org.junit.jupiter.api.Assertions;
3838
import org.junit.jupiter.api.Test;
39+
import org.junit.jupiter.api.Timeout;
3940
import org.mockito.InOrder;
4041
import org.mockito.MockedStatic;
4142
import org.mockito.Mockito;
@@ -44,6 +45,10 @@
4445
import org.xnio.XnioWorker;
4546
import org.xnio.conduits.ConduitStreamSourceChannel;
4647

48+
import java.util.ArrayList;
49+
import java.util.Collections;
50+
import java.util.List;
51+
import java.util.concurrent.CyclicBarrier;
4752
import java.util.concurrent.RejectedExecutionException;
4853

4954
public class ConnectionExceedTest {
@@ -190,4 +195,63 @@ public void testFlightSessionConnectionExceed() throws Exception {
190195
Assertions.assertEquals(2, scheduler.getConnectPoolMgr().getFlightConnectionNum());
191196
}
192197
}
198+
199+
// Two first requests carrying the same bearer token race session creation. getConnectContext
200+
// serializes creation and re-checks the peer-identity index, so exactly one ConnectContext is
201+
// published for the token and every caller gets it. Without that (both build and register one, or
202+
// the loser hits the created-session guard) the pool would hold a second, orphaned session, or a
203+
// caller would fail. Each worker installs its own static mocks because Mockito's mockStatic is
204+
// thread-local; the ConnectScheduler is the one shared instance they publish into.
205+
@Test
206+
@Timeout(60)
207+
public void testConcurrentFirstRequestsPublishExactlyOneSessionPerToken() throws Exception {
208+
ConnectScheduler scheduler = new ConnectScheduler(1000, 100);
209+
Mockito.when(mockEnv.getInternalCatalog()).thenReturn(mockCatalog);
210+
Mockito.when(mockCatalog.getName()).thenReturn("internal");
211+
Mockito.when(mockAuth.getMaxConn(Mockito.anyString())).thenReturn(100L);
212+
Mockito.when(mockEnv.getAuth()).thenReturn(mockAuth);
213+
Mockito.when(mockExecuteEnv.getScheduler()).thenReturn(scheduler);
214+
215+
UserIdentity userIdentity = UserIdentity.createAnalyzedUserIdentWithIp("test_user", "%");
216+
FlightTokenDetails tokenDetails = new FlightTokenDetails(
217+
"shared_token", "test_user",
218+
System.currentTimeMillis(), System.currentTimeMillis() + 3600000,
219+
userIdentity, "127.0.0.1");
220+
Mockito.when(mockTokenManager.validateToken("shared_token")).thenReturn(tokenDetails);
221+
222+
FlightSessionsWithTokenManager manager = new FlightSessionsWithTokenManager(mockTokenManager);
223+
224+
int threadCount = 6;
225+
CyclicBarrier barrier = new CyclicBarrier(threadCount);
226+
List<ConnectContext> results = Collections.synchronizedList(new ArrayList<>());
227+
List<Throwable> errors = Collections.synchronizedList(new ArrayList<>());
228+
List<Thread> threads = new ArrayList<>();
229+
for (int i = 0; i < threadCount; i++) {
230+
Thread t = new Thread(() -> {
231+
try (MockedStatic<ExecuteEnv> execEnv = Mockito.mockStatic(ExecuteEnv.class);
232+
MockedStatic<Env> env = Mockito.mockStatic(Env.class)) {
233+
execEnv.when(ExecuteEnv::getInstance).thenReturn(mockExecuteEnv);
234+
env.when(Env::getCurrentEnv).thenReturn(mockEnv);
235+
barrier.await();
236+
results.add(manager.getConnectContext("shared_token"));
237+
} catch (Throwable e) {
238+
errors.add(e);
239+
}
240+
});
241+
threads.add(t);
242+
t.start();
243+
}
244+
for (Thread t : threads) {
245+
t.join();
246+
}
247+
248+
Assertions.assertTrue(errors.isEmpty(), "getConnectContext threw: " + errors);
249+
Assertions.assertEquals(1, scheduler.getConnectionNum(),
250+
"concurrent first requests on one token must publish exactly one session");
251+
ConnectContext published = scheduler.getContextWithPeerIdentity("shared_token");
252+
Assertions.assertNotNull(published);
253+
for (ConnectContext c : results) {
254+
Assertions.assertSame(published, c, "every caller must get the one published session");
255+
}
256+
}
193257
}

0 commit comments

Comments
 (0)