Skip to content

Commit c96a0a5

Browse files
committed
refactor(python): isolate Windows atomic replace retry
Signed-off-by: Piotr Mlocek <[email protected]>
1 parent 60362d8 commit c96a0a5

2 files changed

Lines changed: 93 additions & 11 deletions

File tree

‎python/openshell/sandbox.py‎

Lines changed: 36 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55

66
import base64
77
import contextlib
8+
import errno
89
import json
910
import os
1011
import pathlib
@@ -1065,6 +1066,40 @@ def _xdg_config_home() -> pathlib.Path:
10651066
# matches `openshell-bootstrap::oidc_token::is_token_expired`.
10661067
_OIDC_TOKEN_EXPIRY_GRACE_SECONDS = 30
10671068

1069+
_IS_WINDOWS = os.name == "nt"
1070+
_WINDOWS_REPLACE_RETRYABLE_ERRORS = frozenset({5, 32})
1071+
_WINDOWS_REPLACE_TIMEOUT_SECONDS = 0.25
1072+
_WINDOWS_REPLACE_INITIAL_DELAY_SECONDS = 0.005
1073+
_WINDOWS_REPLACE_MAX_DELAY_SECONDS = 0.05
1074+
_WINDOWS_REPLACE_LOCK = threading.Lock()
1075+
1076+
1077+
def _atomic_replace(source: pathlib.Path, destination: pathlib.Path) -> None:
1078+
"""Atomically replace a file, retrying transient Windows sharing errors."""
1079+
if not _IS_WINDOWS:
1080+
source.replace(destination)
1081+
return
1082+
1083+
# Serialize writers in this process. The retry still handles other
1084+
# processes (including the Rust CLI) and filesystem scanners that briefly
1085+
# open the destination without delete sharing.
1086+
with _WINDOWS_REPLACE_LOCK:
1087+
deadline = time.monotonic() + _WINDOWS_REPLACE_TIMEOUT_SECONDS
1088+
delay = _WINDOWS_REPLACE_INITIAL_DELAY_SECONDS
1089+
while True:
1090+
try:
1091+
source.replace(destination)
1092+
return
1093+
except PermissionError as error:
1094+
winerror = getattr(error, "winerror", None)
1095+
retryable = winerror in _WINDOWS_REPLACE_RETRYABLE_ERRORS or (
1096+
winerror is None and error.errno == errno.EACCES
1097+
)
1098+
if not retryable or time.monotonic() >= deadline:
1099+
raise
1100+
time.sleep(delay)
1101+
delay = min(delay * 2, _WINDOWS_REPLACE_MAX_DELAY_SECONDS)
1102+
10681103

10691104
def _read_oidc_token_bundle(gateway_dir: pathlib.Path) -> dict | None:
10701105
"""Read and parse `oidc_token.json` for a gateway.
@@ -1531,17 +1566,7 @@ def _write_to_disk(self, bundle: dict) -> None:
15311566
f.write(payload)
15321567
with contextlib.suppress(OSError):
15331568
tmp_path.chmod(0o600)
1534-
for attempt in range(20):
1535-
try:
1536-
tmp_path.replace(path)
1537-
break
1538-
except PermissionError:
1539-
# Concurrent MoveFileExW calls can briefly lock the
1540-
# destination on Windows. Retry that transient sharing
1541-
# violation while preserving POSIX rename behavior.
1542-
if os.name != "nt" or attempt == 19:
1543-
raise
1544-
time.sleep(0.01)
1569+
_atomic_replace(tmp_path, path)
15451570
except BaseException:
15461571
# Clean up our tmp on failure so we don't leave orphaned
15471572
# `.oidc_token.<rand>.tmp` files lying around. The replace

‎python/openshell/sandbox_test.py‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616

1717
import pytest
1818

19+
import openshell.sandbox as sandbox_module
1920
from openshell._proto import openshell_pb2
2021
from openshell.sandbox import (
2122
_PYTHON_CLOUDPICKLE_BOOTSTRAP,
@@ -27,6 +28,7 @@
2728
SandboxRef,
2829
SandboxStatusRef,
2930
TlsConfig,
31+
_atomic_replace,
3032
_BearerAuthInterceptor,
3133
_load_cluster_bearer_token,
3234
_make_cluster_bearer_provider,
@@ -1281,6 +1283,61 @@ def writer(idx: int) -> None:
12811283
r.close()
12821284

12831285

1286+
def test_atomic_replace_retries_windows_sharing_violations(
1287+
tmp_path: Path, monkeypatch: Any
1288+
) -> None:
1289+
source = tmp_path / "source"
1290+
destination = tmp_path / "destination"
1291+
source.write_text("new")
1292+
destination.write_text("old")
1293+
attempts = 0
1294+
delays: list[float] = []
1295+
real_replace = Path.replace
1296+
1297+
def replace(path: Path, target: Path) -> Path:
1298+
nonlocal attempts
1299+
attempts += 1
1300+
if attempts < 3:
1301+
error = PermissionError("destination is busy")
1302+
error.winerror = 32
1303+
raise error
1304+
return real_replace(path, target)
1305+
1306+
monkeypatch.setattr(sandbox_module, "_IS_WINDOWS", True)
1307+
monkeypatch.setattr(Path, "replace", replace)
1308+
monkeypatch.setattr(time, "sleep", delays.append)
1309+
1310+
_atomic_replace(source, destination)
1311+
1312+
assert attempts == 3
1313+
assert delays == [0.005, 0.01]
1314+
assert destination.read_text() == "new"
1315+
1316+
1317+
def test_atomic_replace_does_not_retry_permanent_windows_errors(
1318+
tmp_path: Path, monkeypatch: Any
1319+
) -> None:
1320+
source = tmp_path / "source"
1321+
destination = tmp_path / "destination"
1322+
source.write_text("new")
1323+
attempts = 0
1324+
1325+
def replace(_path: Path, _target: Path) -> Path:
1326+
nonlocal attempts
1327+
attempts += 1
1328+
error = PermissionError("access denied")
1329+
error.winerror = 13
1330+
raise error
1331+
1332+
monkeypatch.setattr(sandbox_module, "_IS_WINDOWS", True)
1333+
monkeypatch.setattr(Path, "replace", replace)
1334+
1335+
with pytest.raises(PermissionError, match="access denied"):
1336+
_atomic_replace(source, destination)
1337+
1338+
assert attempts == 1
1339+
1340+
12841341
def test_sandbox_wrapper_forwards_auth_kwargs_to_from_active_cluster(
12851342
monkeypatch: Any,
12861343
) -> None:

0 commit comments

Comments
 (0)