|
7 | 7 | import json |
8 | 8 | import logging |
9 | 9 | import os |
| 10 | +import tempfile |
| 11 | +import threading |
| 12 | +import time |
10 | 13 | import uuid |
| 14 | +import weakref |
| 15 | +from collections import OrderedDict |
11 | 16 | from collections.abc import Mapping |
12 | 17 | from dataclasses import dataclass, field, fields |
13 | 18 | from datetime import datetime, timezone |
@@ -288,8 +293,37 @@ def __init__( |
288 | 293 | self.storage_path = Path(storage_path) |
289 | 294 | self.storage_path.mkdir(parents=True, exist_ok=True) |
290 | 295 | self._allowed_types: frozenset[str] = frozenset(allowed_checkpoint_types or []) |
| 296 | + # Serialize writes per checkpoint ID within each event loop. Locks are held |
| 297 | + # per (loop, checkpoint_id) because asyncio.Lock is loop-bound; a |
| 298 | + # WeakKeyDictionary keyed by loop lets drops references disappear with the |
| 299 | + # loop. The per-loop dict is LRU-capped so long-running writers with many |
| 300 | + # distinct checkpoint IDs do not grow it without bound. |
| 301 | + self._save_locks_by_loop: weakref.WeakKeyDictionary[ |
| 302 | + asyncio.AbstractEventLoop, OrderedDict[CheckpointID, asyncio.Lock] |
| 303 | + ] = weakref.WeakKeyDictionary() |
| 304 | + self._save_locks_guard = threading.Lock() |
291 | 305 | logger.info(f"Initialized file checkpoint storage at {self.storage_path}") |
292 | 306 |
|
| 307 | + _SAVE_LOCKS_MAX: int = 1024 |
| 308 | + |
| 309 | + def _save_lock_for(self, checkpoint_id: CheckpointID) -> asyncio.Lock: |
| 310 | + """Return the per-event-loop, per-checkpoint-ID lock, creating it on first use.""" |
| 311 | + loop = asyncio.get_running_loop() |
| 312 | + with self._save_locks_guard: |
| 313 | + locks = self._save_locks_by_loop.get(loop) |
| 314 | + if locks is None: |
| 315 | + locks = OrderedDict() |
| 316 | + self._save_locks_by_loop[loop] = locks |
| 317 | + lock = locks.get(checkpoint_id) |
| 318 | + if lock is not None: |
| 319 | + locks.move_to_end(checkpoint_id) |
| 320 | + return lock |
| 321 | + if len(locks) >= self._SAVE_LOCKS_MAX: |
| 322 | + locks.popitem(last=False) |
| 323 | + lock = asyncio.Lock() |
| 324 | + locks[checkpoint_id] = lock |
| 325 | + return lock |
| 326 | + |
293 | 327 | def _validate_file_path(self, checkpoint_id: CheckpointID) -> Path: |
294 | 328 | """Validate that a checkpoint ID resolves to a path within the storage directory. |
295 | 329 |
|
@@ -325,13 +359,49 @@ async def save(self, checkpoint: WorkflowCheckpoint) -> CheckpointID: |
325 | 359 | checkpoint_dict = checkpoint.to_dict() |
326 | 360 | encoded_checkpoint = encode_checkpoint_value(checkpoint_dict) |
327 | 361 |
|
328 | | - def _write_atomic() -> None: |
329 | | - tmp_path = file_path.with_suffix(".json.tmp") |
330 | | - with open(tmp_path, "w") as f: |
331 | | - json.dump(encoded_checkpoint, f, indent=2, ensure_ascii=False) |
332 | | - os.replace(tmp_path, file_path) |
| 362 | + save_lock = self._save_lock_for(checkpoint.checkpoint_id) |
333 | 363 |
|
334 | | - await asyncio.to_thread(_write_atomic) |
| 364 | + def _replace_with_retry(tmp_path: Path) -> None: |
| 365 | + # On Windows, os.replace can transiently fail with PermissionError when a |
| 366 | + # background indexer or AV scan briefly holds a handle to the destination |
| 367 | + # file. Retry the atomic move with a short backoff before giving up. |
| 368 | + for attempt in range(5): |
| 369 | + try: |
| 370 | + os.replace(tmp_path, file_path) |
| 371 | + return |
| 372 | + except PermissionError: |
| 373 | + if attempt == 4: |
| 374 | + raise |
| 375 | + time.sleep(0.001 * (2**attempt)) |
| 376 | + |
| 377 | + def _write_atomic() -> None: |
| 378 | + # Use a unique temp file per save so concurrent saves of the same |
| 379 | + # checkpoint ID do not race on a shared temporary path. The temp file |
| 380 | + # is created in the same directory so os.replace stays atomic. |
| 381 | + tmp_path: Path | None = None |
| 382 | + try: |
| 383 | + with tempfile.NamedTemporaryFile( |
| 384 | + mode="w", |
| 385 | + dir=file_path.parent, |
| 386 | + prefix=f"{file_path.stem}.", |
| 387 | + suffix=".json.tmp", |
| 388 | + delete=False, |
| 389 | + ) as f: |
| 390 | + tmp_path = Path(f.name) |
| 391 | + json.dump(encoded_checkpoint, f, indent=2, ensure_ascii=False) |
| 392 | + _replace_with_retry(tmp_path) |
| 393 | + tmp_path = None |
| 394 | + finally: |
| 395 | + if tmp_path is not None: |
| 396 | + try: |
| 397 | + tmp_path.unlink(missing_ok=True) |
| 398 | + except OSError: |
| 399 | + # Best-effort cleanup only; leaking a temp file is harmless |
| 400 | + # compared to masking the original exception. |
| 401 | + logger.debug(f"Failed to remove checkpoint temp file {tmp_path}", exc_info=True) |
| 402 | + |
| 403 | + async with save_lock: |
| 404 | + await asyncio.to_thread(_write_atomic) |
335 | 405 |
|
336 | 406 | logger.info(f"Saved checkpoint {checkpoint.checkpoint_id} to {file_path}") |
337 | 407 | return checkpoint.checkpoint_id |
|
0 commit comments