Skip to content

Commit 6b2d455

Browse files
AtulJoshi1206copybara-github
authored andcommitted
fix: stop concurrent writes losing app: and user: state
Merge #7942 DatabaseSessionService now serializes, in process, writes to the shared app- and user-state rows from append_event and create_session, and create_session reads the rows its initial state merges into FOR UPDATE on PostgreSQL and MySQL. RedisSessionService.create_session and append_event merge app: and user: state in a watched transaction. Concurrent sessions no longer drop each other's app: or user: keys. PiperOrigin-RevId: 1014248642
1 parent 245f5d5 commit 6b2d455

4 files changed

Lines changed: 426 additions & 79 deletions

File tree

‎src/google/adk/integrations/redis/_redis_session_service.py‎

Lines changed: 38 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,35 @@ def _extract_session_only_state(state: dict[str, Any]) -> dict[str, Any]:
139139
and not k.startswith(State.TEMP_PREFIX)
140140
}
141141

142+
async def _apply_scoped_state_delta(
143+
self, client: Any, key: str, delta: dict[str, Any]
144+
) -> dict[str, Any]:
145+
"""Merges `delta` into the app or user state at `key` and returns it.
146+
147+
Other sessions write the same key, so the read and write run in one
148+
watched transaction that retries if the key changes in between.
149+
"""
150+
if not delta:
151+
raw = await client.get(key)
152+
return cast(dict[str, Any], json.loads(raw)) if raw else {}
153+
154+
merged: dict[str, Any] = {}
155+
156+
async def merge(pipe: Any) -> None:
157+
nonlocal merged
158+
raw = await pipe.get(key)
159+
merged = json.loads(raw) if raw else {}
160+
merged.update(delta)
161+
pipe.multi()
162+
pipe.set(
163+
key,
164+
json.dumps(merged),
165+
ex=self.config.ttl_seconds if self.config.ttl_seconds > 0 else None,
166+
)
167+
168+
await client.transaction(merge, key)
169+
return merged
170+
142171
async def create_session(
143172
self,
144173
*,
@@ -199,28 +228,10 @@ async def create_session(
199228
f"Session {sid} already exists for user {user_id} in app {app_name}."
200229
)
201230

202-
app_raw, user_raw = await asyncio.gather(
203-
client.get(app_key),
204-
client.get(user_key),
231+
current_app, current_user = await asyncio.gather(
232+
self._apply_scoped_state_delta(client, app_key, app_state_delta),
233+
self._apply_scoped_state_delta(client, user_key, user_state_delta),
205234
)
206-
current_app = json.loads(app_raw) if app_raw else {}
207-
current_user = json.loads(user_raw) if user_raw else {}
208-
209-
if app_state_delta:
210-
current_app.update(app_state_delta)
211-
await client.set(
212-
app_key,
213-
json.dumps(current_app),
214-
ex=self.config.ttl_seconds if self.config.ttl_seconds > 0 else None,
215-
)
216-
217-
if user_state_delta:
218-
current_user.update(user_state_delta)
219-
await client.set(
220-
user_key,
221-
json.dumps(current_user),
222-
ex=self.config.ttl_seconds if self.config.ttl_seconds > 0 else None,
223-
)
224235

225236
# Return session with dynamically merged state (including app/user/temp state)
226237
merged_state = self._merge_state(current_app, current_user, session_state)
@@ -376,25 +387,14 @@ async def append_event(self, session: Session, event: Event) -> Event:
376387
session_delta = deltas.get("session", {})
377388

378389
if app_delta:
379-
app_key = self._app_state_key(session.app_name)
380-
app_raw = await client.get(app_key)
381-
app_state = json.loads(app_raw) if app_raw else {}
382-
app_state.update(app_delta)
383-
await client.set(
384-
app_key,
385-
json.dumps(app_state),
386-
ex=self.config.ttl_seconds if self.config.ttl_seconds > 0 else None,
390+
await self._apply_scoped_state_delta(
391+
client, self._app_state_key(session.app_name), app_delta
387392
)
388-
389393
if user_delta:
390-
user_key = self._user_state_key(session.app_name, session.user_id)
391-
user_raw = await client.get(user_key)
392-
user_state = json.loads(user_raw) if user_raw else {}
393-
user_state.update(user_delta)
394-
await client.set(
395-
user_key,
396-
json.dumps(user_state),
397-
ex=self.config.ttl_seconds if self.config.ttl_seconds > 0 else None,
394+
await self._apply_scoped_state_delta(
395+
client,
396+
self._user_state_key(session.app_name, session.user_id),
397+
user_delta,
398398
)
399399

400400
# Prepare storage copy with ONLY session-scoped state (excluding app:, user:, temp:)

‎src/google/adk/sessions/database_session_service.py‎

Lines changed: 104 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
import asyncio
1717
from collections.abc import Callable
1818
from contextlib import asynccontextmanager
19+
from contextlib import AsyncExitStack
1920
import copy
2021
from datetime import datetime
2122
from datetime import timezone
@@ -113,9 +114,9 @@
113114
_MYSQL_DIALECT: "aiomysql",
114115
_MARIADB_DIALECT: "asyncmy",
115116
}
116-
# Tuple key order for in-process per-session lock maps:
117-
# (app_name, user_id, session_id).
118-
_SessionLockKey: TypeAlias = tuple[str, str, str]
117+
# In-process lock keys: ("session", app_name, user_id, session_id),
118+
# ("user", app_name, user_id) or ("app", app_name).
119+
_LockKey: TypeAlias = tuple[str, ...]
119120
_StorageState: TypeAlias = (
120121
StorageAppStateV0
121122
| StorageAppStateV1
@@ -207,14 +208,20 @@ async def _get_or_create_state(
207208
state_model: type[_StorageStateT],
208209
primary_key: Any,
209210
defaults: dict[str, Any],
211+
use_row_level_locking: bool = False,
210212
) -> _StorageStateT:
211213
"""Returns an existing state row or creates one, handling concurrent inserts.
212214
213215
Uses a SAVEPOINT so that an IntegrityError from a racing INSERT does not
214-
invalidate the outer transaction.
216+
invalidate the outer transaction. With `use_row_level_locking`, the returned
217+
row is locked `FOR UPDATE`.
215218
"""
216219
row: _StorageStateT | None = await sql_session.get(state_model, primary_key)
217220
if row is not None:
221+
if use_row_level_locking:
222+
# Lock only an existing row: on MySQL a locking read of a missing key
223+
# takes a gap lock, and two such inserts of the same key deadlock.
224+
await sql_session.refresh(row, with_for_update=True)
218225
return row
219226
try:
220227
async with sql_session.begin_nested():
@@ -225,7 +232,11 @@ async def _get_or_create_state(
225232
except IntegrityError:
226233
# Another concurrent caller inserted the row first.
227234
# The savepoint was rolled back, so re-fetch the winner's row.
228-
row = await sql_session.get(state_model, primary_key)
235+
row = await sql_session.get(
236+
state_model,
237+
primary_key,
238+
with_for_update=True if use_row_level_locking else None,
239+
)
229240
if row is None:
230241
raise
231242
return row
@@ -459,10 +470,11 @@ def __init__(
459470
# The current database schema version in use, "None" if not yet checked
460471
self._db_schema_version: str | None = None
461472

462-
# Per-session locks used to serialize append_event calls in this process.
463-
self._session_locks: dict[_SessionLockKey, asyncio.Lock] = {}
464-
self._session_lock_ref_count: dict[_SessionLockKey, int] = {}
465-
self._session_locks_guard = asyncio.Lock()
473+
# In-process locks for appends to one session and for writes to the
474+
# shared app- and user-state rows.
475+
self._locks: dict[_LockKey, asyncio.Lock] = {}
476+
self._lock_ref_count: dict[_LockKey, int] = {}
477+
self._locks_guard = asyncio.Lock()
466478

467479
def _get_schema_classes(self) -> _SchemaClasses:
468480
return _SchemaClasses(self._db_schema_version)
@@ -510,36 +522,64 @@ def _uses_naive_datetime(self) -> bool:
510522
return self.db_engine.dialect.name in _NAIVE_DATETIME_DIALECTS
511523

512524
@asynccontextmanager
513-
async def _with_session_lock(
514-
self, *, app_name: str, user_id: str, session_id: str
515-
) -> AsyncIterator[None]:
516-
"""Serializes event appends for the same session within this process."""
517-
# Use one lock per logical ADK session to prevent concurrent append_event
518-
# writes from racing in the same process.
519-
lock_key = (app_name, user_id, session_id)
520-
async with self._session_locks_guard:
521-
lock = self._session_locks.get(lock_key)
525+
async def _with_lock(self, lock_key: _LockKey) -> AsyncIterator[None]:
526+
"""Holds the in-process lock for `lock_key`, creating it on first use."""
527+
async with self._locks_guard:
528+
lock = self._locks.get(lock_key)
522529
if lock is None:
523530
lock = asyncio.Lock()
524-
self._session_locks[lock_key] = lock
531+
self._locks[lock_key] = lock
525532
# Reference counting keeps lock objects alive while they are in use by
526533
# concurrent tasks and allows cleanup once all waiters complete.
527-
self._session_lock_ref_count[lock_key] = (
528-
self._session_lock_ref_count.get(lock_key, 0) + 1
529-
)
534+
self._lock_ref_count[lock_key] = self._lock_ref_count.get(lock_key, 0) + 1
530535

531536
try:
532537
async with lock:
533538
yield
534539
finally:
535-
async with self._session_locks_guard:
536-
remaining = self._session_lock_ref_count.get(lock_key, 0) - 1
540+
async with self._locks_guard:
541+
remaining = self._lock_ref_count.get(lock_key, 0) - 1
537542
# Remove lock bookkeeping after the last waiter exits.
538543
if remaining <= 0 and not lock.locked():
539-
self._session_lock_ref_count.pop(lock_key, None)
540-
self._session_locks.pop(lock_key, None)
544+
self._lock_ref_count.pop(lock_key, None)
545+
self._locks.pop(lock_key, None)
541546
else:
542-
self._session_lock_ref_count[lock_key] = remaining
547+
self._lock_ref_count[lock_key] = remaining
548+
549+
@asynccontextmanager
550+
async def _with_session_lock(
551+
self, *, app_name: str, user_id: str, session_id: str
552+
) -> AsyncIterator[None]:
553+
"""Serializes event appends for the same session within this process."""
554+
# Use one lock per logical ADK session to prevent concurrent append_event
555+
# writes from racing in the same process.
556+
async with self._with_lock(("session", app_name, user_id, session_id)):
557+
yield
558+
559+
@asynccontextmanager
560+
async def _with_shared_state_locks(
561+
self,
562+
*,
563+
app_name: str,
564+
user_id: str,
565+
app_state: bool,
566+
user_state: bool,
567+
) -> AsyncIterator[None]:
568+
"""Serializes writes to the shared app- and user-state rows in this process.
569+
570+
Writers merge their delta into the stored JSON and write it back, so
571+
unordered writers drop each other's keys. Dialects without row-level
572+
locking (SQLite) have only this lock to order them. Take it after any
573+
session lock; the app lock always precedes the user lock to avoid cycles.
574+
"""
575+
async with AsyncExitStack() as stack:
576+
if app_state:
577+
await stack.enter_async_context(self._with_lock(("app", app_name)))
578+
if user_state:
579+
await stack.enter_async_context(
580+
self._with_lock(("user", app_name, user_id))
581+
)
582+
yield
543583

544584
async def prepare_tables(self) -> None:
545585
"""Ensure database tables are ready for use.
@@ -670,7 +710,25 @@ async def create_session(
670710
if not session_id:
671711
session_id = platform_uuid.new_uuid()
672712
schema = self._get_schema_classes()
673-
async with self._rollback_on_exception_session() as sql_session:
713+
use_row_level_locking = self._supports_row_level_locking()
714+
715+
# Extract state deltas
716+
state_deltas = _session_util.extract_json_safe_state_delta(state or {})
717+
app_state_delta = state_deltas["app"]
718+
user_state_delta = state_deltas["user"]
719+
session_state = state_deltas["session"]
720+
has_app_delta = bool(app_state_delta)
721+
has_user_delta = bool(user_state_delta)
722+
723+
async with (
724+
self._with_shared_state_locks(
725+
app_name=app_name,
726+
user_id=user_id,
727+
app_state=has_app_delta,
728+
user_state=has_user_delta,
729+
),
730+
self._rollback_on_exception_session() as sql_session,
731+
):
674732
if has_user_provided_id:
675733
existing_session = await sql_session.get(
676734
schema.StorageSession, (app_name, user_id, session_id)
@@ -679,30 +737,27 @@ async def create_session(
679737
raise AlreadyExistsError(
680738
f"Session with id {session_id} already exists."
681739
)
682-
# Get or create state rows, handling concurrent insert races.
740+
# Get or create state rows, handling concurrent insert races. Rows the
741+
# initial state merges into are locked, as in append_event.
683742
storage_app_state = await _get_or_create_state(
684743
sql_session=sql_session,
685744
state_model=schema.StorageAppState,
686745
primary_key=app_name,
687746
defaults={"app_name": app_name, "state": {}},
747+
use_row_level_locking=use_row_level_locking and has_app_delta,
688748
)
689749
storage_user_state = await _get_or_create_state(
690750
sql_session=sql_session,
691751
state_model=schema.StorageUserState,
692752
primary_key=(app_name, user_id),
693753
defaults={"app_name": app_name, "user_id": user_id, "state": {}},
754+
use_row_level_locking=use_row_level_locking and has_user_delta,
694755
)
695756

696-
# Extract state deltas
697-
state_deltas = _session_util.extract_json_safe_state_delta(state or {})
698-
app_state_delta = state_deltas["app"]
699-
user_state_delta = state_deltas["user"]
700-
session_state = state_deltas["session"]
701-
702757
# Apply state delta
703-
if app_state_delta:
758+
if has_app_delta:
704759
storage_app_state.state.update(app_state_delta)
705-
if user_state_delta:
760+
if has_user_delta:
706761
storage_user_state.state.update(user_state_delta)
707762

708763
# Store the session
@@ -941,10 +996,18 @@ async def append_event(self, session: Session, event: Event) -> Event:
941996
has_app_delta = bool(state_deltas["app"])
942997
has_user_delta = bool(state_deltas["user"])
943998

944-
async with self._with_session_lock(
945-
app_name=session.app_name,
946-
user_id=session.user_id,
947-
session_id=session.id,
999+
async with (
1000+
self._with_session_lock(
1001+
app_name=session.app_name,
1002+
user_id=session.user_id,
1003+
session_id=session.id,
1004+
),
1005+
self._with_shared_state_locks(
1006+
app_name=session.app_name,
1007+
user_id=session.user_id,
1008+
app_state=has_app_delta,
1009+
user_state=has_user_delta,
1010+
),
9481011
):
9491012
async with self._rollback_on_exception_session() as sql_session:
9501013
storage_session_stmt = (

0 commit comments

Comments
 (0)