Skip to content

Commit fd0a15a

Browse files
Python: fix: make concurrent FileCheckpointStorage saves not race on a shared temp path
FileCheckpointStorage.save() wrote to and then renamed a fixed "<checkpoint-id>.json.tmp" path, so concurrent saves of the same checkpoint ID raced over the shared temp file. Whichever save renamed first removed the temp file still being written by another save, which then failed with FileNotFoundError / PermissionError in os.replace. Create a unique temp file per save in the destination directory (so os.replace remains atomic), serialize same-ID writes with a per-ID lock, and retry the atomic move briefly to absorb the transient Windows background-handle PermissionError that surfaces even for fully serialized replaces. Fixes #7748
1 parent da11dae commit fd0a15a

2 files changed

Lines changed: 106 additions & 6 deletions

File tree

‎python/packages/core/agent_framework/_workflows/_checkpoint.py‎

Lines changed: 76 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,12 @@
77
import json
88
import logging
99
import os
10+
import tempfile
11+
import threading
12+
import time
1013
import uuid
14+
import weakref
15+
from collections import OrderedDict
1116
from collections.abc import Mapping
1217
from dataclasses import dataclass, field, fields
1318
from datetime import datetime, timezone
@@ -288,8 +293,37 @@ def __init__(
288293
self.storage_path = Path(storage_path)
289294
self.storage_path.mkdir(parents=True, exist_ok=True)
290295
self._allowed_types: frozenset[str] = frozenset(allowed_checkpoint_types or [])
296+
# Serialize writes per checkpoint ID within each event loop. Locks are held
297+
# per (loop, checkpoint_id) because asyncio.Lock is loop-bound; a
298+
# WeakKeyDictionary keyed by loop lets drops references disappear with the
299+
# loop. The per-loop dict is LRU-capped so long-running writers with many
300+
# distinct checkpoint IDs do not grow it without bound.
301+
self._save_locks_by_loop: weakref.WeakKeyDictionary[
302+
asyncio.AbstractEventLoop, OrderedDict[CheckpointID, asyncio.Lock]
303+
] = weakref.WeakKeyDictionary()
304+
self._save_locks_guard = threading.Lock()
291305
logger.info(f"Initialized file checkpoint storage at {self.storage_path}")
292306

307+
_SAVE_LOCKS_MAX: int = 1024
308+
309+
def _save_lock_for(self, checkpoint_id: CheckpointID) -> asyncio.Lock:
310+
"""Return the per-event-loop, per-checkpoint-ID lock, creating it on first use."""
311+
loop = asyncio.get_running_loop()
312+
with self._save_locks_guard:
313+
locks = self._save_locks_by_loop.get(loop)
314+
if locks is None:
315+
locks = OrderedDict()
316+
self._save_locks_by_loop[loop] = locks
317+
lock = locks.get(checkpoint_id)
318+
if lock is not None:
319+
locks.move_to_end(checkpoint_id)
320+
return lock
321+
if len(locks) >= self._SAVE_LOCKS_MAX:
322+
locks.popitem(last=False)
323+
lock = asyncio.Lock()
324+
locks[checkpoint_id] = lock
325+
return lock
326+
293327
def _validate_file_path(self, checkpoint_id: CheckpointID) -> Path:
294328
"""Validate that a checkpoint ID resolves to a path within the storage directory.
295329
@@ -325,13 +359,49 @@ async def save(self, checkpoint: WorkflowCheckpoint) -> CheckpointID:
325359
checkpoint_dict = checkpoint.to_dict()
326360
encoded_checkpoint = encode_checkpoint_value(checkpoint_dict)
327361

328-
def _write_atomic() -> None:
329-
tmp_path = file_path.with_suffix(".json.tmp")
330-
with open(tmp_path, "w") as f:
331-
json.dump(encoded_checkpoint, f, indent=2, ensure_ascii=False)
332-
os.replace(tmp_path, file_path)
362+
save_lock = self._save_lock_for(checkpoint.checkpoint_id)
333363

334-
await asyncio.to_thread(_write_atomic)
364+
def _replace_with_retry(tmp_path: Path) -> None:
365+
# On Windows, os.replace can transiently fail with PermissionError when a
366+
# background indexer or AV scan briefly holds a handle to the destination
367+
# file. Retry the atomic move with a short backoff before giving up.
368+
for attempt in range(5):
369+
try:
370+
os.replace(tmp_path, file_path)
371+
return
372+
except PermissionError:
373+
if attempt == 4:
374+
raise
375+
time.sleep(0.001 * (2**attempt))
376+
377+
def _write_atomic() -> None:
378+
# Use a unique temp file per save so concurrent saves of the same
379+
# checkpoint ID do not race on a shared temporary path. The temp file
380+
# is created in the same directory so os.replace stays atomic.
381+
tmp_path: Path | None = None
382+
try:
383+
with tempfile.NamedTemporaryFile(
384+
mode="w",
385+
dir=file_path.parent,
386+
prefix=f"{file_path.stem}.",
387+
suffix=".json.tmp",
388+
delete=False,
389+
) as f:
390+
tmp_path = Path(f.name)
391+
json.dump(encoded_checkpoint, f, indent=2, ensure_ascii=False)
392+
_replace_with_retry(tmp_path)
393+
tmp_path = None
394+
finally:
395+
if tmp_path is not None:
396+
try:
397+
tmp_path.unlink(missing_ok=True)
398+
except OSError:
399+
# Best-effort cleanup only; leaking a temp file is harmless
400+
# compared to masking the original exception.
401+
logger.debug(f"Failed to remove checkpoint temp file {tmp_path}", exc_info=True)
402+
403+
async with save_lock:
404+
await asyncio.to_thread(_write_atomic)
335405

336406
logger.info(f"Saved checkpoint {checkpoint.checkpoint_id} to {file_path}")
337407
return checkpoint.checkpoint_id

‎python/packages/core/tests/workflow/test_checkpoint.py‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
# Copyright (c) Microsoft. All rights reserved.
22

3+
import asyncio
34
import json
45
import tempfile
56
from dataclasses import dataclass
@@ -1115,6 +1116,35 @@ async def test_file_checkpoint_storage_save_and_load():
11151116
assert loaded_checkpoint.pending_request_info_events == checkpoint.pending_request_info_events
11161117

11171118

1119+
async def test_file_checkpoint_storage_concurrent_saves_same_id():
1120+
"""Concurrent saves of the same checkpoint ID must not fail on a shared temp path.
1121+
1122+
Regression for https://github.com/microsoft/agent-framework/issues/7748:
1123+
FileCheckpointStorage.save() used a fixed `<id>.json.tmp` temp path, so concurrent
1124+
saves raced on it (one rename removed it before another's rename). Uses enough
1125+
concurrent saves to reliably trip the race on the unfixed code.
1126+
"""
1127+
with tempfile.TemporaryDirectory() as temp_dir:
1128+
storage = FileCheckpointStorage(temp_dir)
1129+
checkpoint = WorkflowCheckpoint(
1130+
workflow_name="test-workflow",
1131+
graph_signature_hash="test-hash",
1132+
checkpoint_id="shared-id",
1133+
)
1134+
1135+
results = await asyncio.gather(*(storage.save(checkpoint) for _ in range(50)), return_exceptions=True)
1136+
1137+
errors = [r for r in results if isinstance(r, BaseException)]
1138+
assert not errors, f"concurrent saves raised internal filesystem errors: {errors[:1]!r}"
1139+
assert all(r == "shared-id" for r in results)
1140+
# One of the saves won; the destination is intact and parseable, not corrupted or truncated.
1141+
assert (Path(temp_dir) / "shared-id.json").exists()
1142+
loaded = await storage.load("shared-id")
1143+
assert loaded.checkpoint_id == checkpoint.checkpoint_id
1144+
assert loaded.workflow_name == checkpoint.workflow_name
1145+
assert loaded.graph_signature_hash == checkpoint.graph_signature_hash
1146+
1147+
11181148
async def test_file_checkpoint_storage_load_nonexistent():
11191149
with tempfile.TemporaryDirectory() as temp_dir:
11201150
storage = FileCheckpointStorage(temp_dir)

0 commit comments

Comments
 (0)