Skip to content

Commit e8d95c4

Browse files
committed
fix(sts): bound the ADK token cache per session
One session presenting many distinct credentials could fill the whole cache and evict the entries of unrelated sessions, whose runs cannot re-mint them before their next tool call. Cap each session at MAX_ENTRIES_PER_SESSION before the whole-cache bound, and report a capacity eviction at warning level since it drops valid credentials. Signed-off-by: QuentinBisson <[email protected]>
1 parent 5a396bd commit e8d95c4

2 files changed

Lines changed: 106 additions & 29 deletions

File tree

‎python/packages/agentsts-adk/src/agentsts/adk/_base.py‎

Lines changed: 46 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
import inspect
66
import logging
77
import time
8-
from typing import Awaitable, Callable, Dict, List, Optional, Union
8+
from typing import Awaitable, Callable, Dict, List, Optional, Tuple, Union
99

1010
import jwt
1111
from agentsts.core import STSIntegrationBase, TokenType
@@ -38,8 +38,14 @@
3838
# only, never the caller's authority, and a cache hit renews it.
3939
MAX_CACHE_TTL_SECONDS = 300
4040

41-
# Upper bound on cache entries. The sweep runs between runs, so concurrent runs
42-
# holding many distinct credentials need a bound that does not wait for one.
41+
# Upper bound on the entries one session may hold. A session carries as many
42+
# entries as it has distinct callers, so this bound is what keeps one session's
43+
# burst of credentials from evicting another session's.
44+
MAX_ENTRIES_PER_SESSION = 32
45+
46+
# Upper bound on cache entries, across sessions. The sweep runs between runs, so
47+
# concurrent runs holding many distinct credentials need a bound that does not
48+
# wait for one.
4349
MAX_CACHE_ENTRIES = 1024
4450

4551
# Separates the session part of a cache key from the subject part. It cannot
@@ -321,6 +327,7 @@ async def before_run_callback(
321327
self.token_cache[cache_key] = entry
322328
self._touch(entry)
323329
self._earliest_eviction = _earlier_expiry(self._earliest_eviction, entry.evict_after)
330+
self._evict_over_session_capacity(invocation_context.session.id)
324331
self._evict_over_capacity()
325332
logger.debug("Cached new subject token")
326333
return None
@@ -384,25 +391,51 @@ def _touch(self, entry: _TokenCacheEntry) -> None:
384391
if entry.expiry is None:
385392
entry.evict_after = int(time.time()) + MAX_CACHE_TTL_SECONDS
386393

394+
def _evict_over_session_capacity(self, session_id: str) -> None:
395+
"""Drop the session's least recently used entries once it is over capacity.
396+
397+
The bound is per session so that a session presenting many distinct
398+
credentials evicts its own entries and not those of unrelated sessions.
399+
The whole cache is walked because entries are not indexed by session,
400+
and only once the session can be over capacity at all.
401+
"""
402+
if len(self.token_cache) <= MAX_ENTRIES_PER_SESSION:
403+
return
404+
405+
prefix = f"{session_id}{_KEY_SEPARATOR}"
406+
entries = [item for item in self.token_cache.items() if item[0].startswith(prefix)]
407+
self._evict_least_recently_used(entries, len(entries) - MAX_ENTRIES_PER_SESSION, "session")
408+
387409
def _evict_over_capacity(self) -> None:
388410
"""Drop the least recently used entries once the cache is over capacity.
389411
390-
The sweep runs between runs only, so it cannot bound a run that sees
391-
many distinct credentials while it holds them. The entry the current run
392-
just cached is the most recently used, so it is never dropped here.
412+
The sweep runs between runs only, so it cannot bound runs that see many
413+
distinct credentials while they hold them.
414+
"""
415+
self._evict_least_recently_used(
416+
list(self.token_cache.items()), len(self.token_cache) - MAX_CACHE_ENTRIES, "cache"
417+
)
393418

394-
Use order is the only signal available here, so an entry a run still
395-
holds is dropped once MAX_CACHE_ENTRIES more recent callers arrive
396-
between two of its tool calls. That run then injects no credential for
397-
the rest of its tool calls.
419+
def _evict_least_recently_used(
420+
self, entries: List[Tuple[str, _TokenCacheEntry]], overflow: int, bound: str
421+
) -> None:
422+
"""Drop the overflow least recently used entries among entries.
423+
424+
Unlike the sweep, this drops entries whose credentials are still valid.
425+
Use order is the only signal available, so an entry a run still holds is
426+
dropped once enough more recent callers arrive between two of its tool
427+
calls, and that run injects no credential for the rest of its tool
428+
calls. It is reported above debug for that reason. The entry the current
429+
run just cached is the most recently used, so it is never dropped here.
398430
"""
399-
overflow = len(self.token_cache) - MAX_CACHE_ENTRIES
400431
if overflow <= 0:
401432
return
402433

403-
for key, _ in heapq.nsmallest(overflow, self.token_cache.items(), key=lambda item: item[1].last_used):
434+
for key, _ in heapq.nsmallest(overflow, entries, key=lambda item: item[1].last_used):
404435
del self.token_cache[key]
405-
logger.debug(f"Dropped {overflow} cached token(s) over the cache capacity")
436+
logger.warning(
437+
f"Dropped {overflow} valid cached token(s) to stay within the {bound} capacity; callers may re-exchange"
438+
)
406439

407440
def cache_key(self, invocation_context: InvocationContext) -> Optional[str]:
408441
"""Key the cache on the session and the acting subject, so a session

‎python/packages/agentsts-adk/tests/test_adk_integration.py‎

Lines changed: 60 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
from google.adk.tools.mcp_tool.mcp_toolset import MCPToolset
1111

1212
from agentsts.adk import ADKSTSIntegration, ADKTokenPropagationPlugin
13-
from agentsts.adk._base import HEADERS_KEY, MAX_CACHE_ENTRIES, MAX_CACHE_TTL_SECONDS
13+
from agentsts.adk._base import HEADERS_KEY, MAX_CACHE_ENTRIES, MAX_CACHE_TTL_SECONDS, MAX_ENTRIES_PER_SESSION
1414
from agentsts.adk._base import _cache_key as build_cache_key
1515
from agentsts.adk._base import _extract_jwt_expiry as extract_jwt_expiry
1616
from agentsts.adk._base import _extract_jwt_from_headers as extract_jwt_from_headers
@@ -1335,27 +1335,41 @@ async def test_a_concurrent_run_does_not_sweep_an_in_flight_caller(self):
13351335
assert plugin.header_provider(in_flight_ctx) == {"Authorization": "Bearer delegated-opaque-alice"}
13361336

13371337
@pytest.mark.asyncio
1338-
async def test_capacity_eviction_keeps_the_entry_of_an_in_flight_caller(self):
1338+
@pytest.mark.parametrize(
1339+
("burst_session", "capacity"),
1340+
[
1341+
pytest.param(lambda index: "sess-burst", MAX_ENTRIES_PER_SESSION, id="session"),
1342+
pytest.param(lambda index: f"sess-burst-{index}", MAX_CACHE_ENTRIES, id="cache"),
1343+
],
1344+
)
1345+
async def test_capacity_eviction_keeps_the_entry_of_an_in_flight_caller(self, burst_session, capacity):
13391346
"""Case: a burst of distinct credentials must not evict the entry a run is
1340-
still making tool calls with."""
1347+
still making tool calls with, under either capacity bound."""
13411348
plugin = ADKTokenPropagationPlugin(sts_integration=None)
13421349

13431350
in_flight = self._make_invocation_context("sess-burst", headers={"Authorization": "Bearer opaque-alice"})
13441351
await plugin.before_run_callback(invocation_context=in_flight)
13451352
in_flight_ctx = self._make_readonly_context(in_flight)
13461353

1347-
for index in range(MAX_CACHE_ENTRIES + 25):
1354+
for index in range(capacity + 25):
13481355
# Each new caller is more recent than the last tool call of the
13491356
# in-flight run, so only a renewed entry survives the burst.
13501357
assert plugin.header_provider(in_flight_ctx) == {"Authorization": "Bearer opaque-alice"}
1351-
ic = self._make_invocation_context("sess-burst", headers={"Authorization": f"Bearer token-{index}"})
1358+
ic = self._make_invocation_context(burst_session(index), headers={"Authorization": f"Bearer token-{index}"})
13521359
await plugin.before_run_callback(invocation_context=ic)
13531360

1354-
assert len(plugin.token_cache) == MAX_CACHE_ENTRIES
1361+
assert len(plugin.token_cache) == capacity
13551362
assert plugin.header_provider(in_flight_ctx) == {"Authorization": "Bearer opaque-alice"}
13561363

13571364
@pytest.mark.asyncio
1358-
async def test_capacity_eviction_drops_a_caller_that_makes_no_tool_call(self):
1365+
@pytest.mark.parametrize(
1366+
("burst_session", "capacity"),
1367+
[
1368+
pytest.param(lambda index: "sess-idle-burst", MAX_ENTRIES_PER_SESSION, id="session"),
1369+
pytest.param(lambda index: f"sess-idle-burst-{index}", MAX_CACHE_ENTRIES, id="cache"),
1370+
],
1371+
)
1372+
async def test_capacity_eviction_drops_a_caller_that_makes_no_tool_call(self, burst_session, capacity):
13591373
"""Case: use order is the only signal the capacity bound has, so a run
13601374
that pauses long enough loses its entry and injects no credential.
13611375
@@ -1370,11 +1384,11 @@ async def test_capacity_eviction_drops_a_caller_that_makes_no_tool_call(self):
13701384
assert plugin.header_provider(idle_ctx) == {"Authorization": "Bearer opaque-alice"}
13711385

13721386
# The run waits on a model turn, so it renews nothing while the burst runs.
1373-
for index in range(MAX_CACHE_ENTRIES + 5):
1374-
ic = self._make_invocation_context("sess-idle-burst", headers={"Authorization": f"Bearer token-{index}"})
1387+
for index in range(capacity + 5):
1388+
ic = self._make_invocation_context(burst_session(index), headers={"Authorization": f"Bearer token-{index}"})
13751389
await plugin.before_run_callback(invocation_context=ic)
13761390

1377-
assert len(plugin.token_cache) == MAX_CACHE_ENTRIES
1391+
assert len(plugin.token_cache) == capacity
13781392
assert plugin.cache_key(idle) not in plugin.token_cache
13791393
assert plugin.header_provider(idle_ctx) == {}
13801394

@@ -1528,19 +1542,49 @@ async def test_missing_authorization_header_is_reported_once(self):
15281542
assert mock_logger.warning.call_args_list == []
15291543

15301544
@pytest.mark.asyncio
1531-
async def test_cache_is_capped_by_entry_count(self):
1532-
"""Case: one session seeing many distinct credentials is capped, since the
1533-
sweep never runs while a single run holds them."""
1545+
@pytest.mark.parametrize(
1546+
("burst_session", "capacity"),
1547+
[
1548+
pytest.param(lambda index: "one-session", MAX_ENTRIES_PER_SESSION, id="session"),
1549+
pytest.param(lambda index: f"session-{index}", MAX_CACHE_ENTRIES, id="cache"),
1550+
],
1551+
)
1552+
async def test_cache_is_capped_by_entry_count(self, burst_session, capacity):
1553+
"""Case: many distinct credentials are capped, since the sweep never runs
1554+
while the runs hold them."""
15341555
plugin = ADKTokenPropagationPlugin(sts_integration=None)
15351556

1536-
for index in range(MAX_CACHE_ENTRIES + 25):
1537-
ic = self._make_invocation_context("one-session", headers={"Authorization": f"Bearer token-{index}"})
1557+
for index in range(capacity + 25):
1558+
ic = self._make_invocation_context(burst_session(index), headers={"Authorization": f"Bearer token-{index}"})
15381559
await plugin.before_run_callback(invocation_context=ic)
15391560

1540-
assert len(plugin.token_cache) == MAX_CACHE_ENTRIES
1561+
assert len(plugin.token_cache) == capacity
15411562
# The run that just cached keeps its own entry.
15421563
assert plugin.cache_key(ic) in plugin.token_cache
15431564

1565+
@pytest.mark.asyncio
1566+
async def test_a_session_burst_does_not_evict_another_sessions_entry(self):
1567+
"""Case: one session presenting many distinct credentials evicts its own
1568+
entries, not those of an unrelated session.
1569+
1570+
The victim run is between tool calls, so it renews nothing during the
1571+
burst. before_run_callback has already run, so a lost entry would leave
1572+
the rest of that run with no delegated identity.
1573+
"""
1574+
plugin = ADKTokenPropagationPlugin(sts_integration=None)
1575+
1576+
victim = self._make_invocation_context("sess-victim", headers={"Authorization": "Bearer opaque-alice"})
1577+
await plugin.before_run_callback(invocation_context=victim)
1578+
victim_ctx = self._make_readonly_context(victim)
1579+
assert plugin.header_provider(victim_ctx) == {"Authorization": "Bearer opaque-alice"}
1580+
1581+
for index in range(MAX_CACHE_ENTRIES + 25):
1582+
ic = self._make_invocation_context("sess-noisy", headers={"Authorization": f"Bearer token-{index}"})
1583+
await plugin.before_run_callback(invocation_context=ic)
1584+
1585+
assert len(plugin.token_cache) == MAX_ENTRIES_PER_SESSION + 1
1586+
assert plugin.header_provider(victim_ctx) == {"Authorization": "Bearer opaque-alice"}
1587+
15441588
def test_header_provider_without_context_fails_closed(self):
15451589
"""Case: a tool call with no invocation context gets no header instead of raising."""
15461590
plugin = ADKTokenPropagationPlugin(sts_integration=None)

0 commit comments

Comments
 (0)