Skip to content

Commit aefa482

Browse files
ppcvoteclaude
andauthored
fix(agent-cli): Windows temp-cwd cleanup must not fail a successful batch (NVIDIA#317)
On Windows, run_agent_cli's TemporaryDirectory raised WinError 32 at __exit__ whenever any live process still held the per-invocation temp cwd — and the agent CLI's process tree routinely does: the binary is a .cmd shim whose node children can outlive the direct child, and a killed-but-unreaped child on the timeout path holds its cwd too. A directory that is any live process's working directory (or contains any open handle) cannot be removed on Windows, so cleanup raised *after* the model had already answered, and llm_analyzer_base recorded the batch as llm_batch_failed. Under batch concurrency the teardown window widens, which is why multi-file scans failed while single-file scans passed (NVIDIA#315). Each invocation already has a unique mkdtemp directory, so this was never a cross-worker path collision; it is delete-at-exit racing the process tree's teardown. Fix: mkdtemp + explicit best-effort cleanup. _cleanup_temp_dir retries briefly (10 x 0.2s) to reclaim the directory once the holder exits, then leaks it with a warning instead of raising. A cleanup failure never outranks a successful response. Verified on Windows 10 with a mechanism-faithful fake CLI (a .cmd shim that detaches a grandchild holding the temp cwd): 8 concurrent batches fail 8/8 with the exact WinError 32 signature from NVIDIA#315 on main, and pass 8/8 with this change. Controlled experiments confirm both hold modes (another process's cwd; an open file handle inside the dir) block rmtree with WinError 32. Tests: 5 new cases including a Windows-only real-handle hold and a regression test asserting run_agent_cli returns the response when rmtree keeps failing. tests/unit/test_agent_cli.py 87/87 on Windows. Closes NVIDIA#315 Signed-off-by: ppcvote <[email protected]> Co-authored-by: Claude Fable 5 <[email protected]>
1 parent bfdcd5e commit aefa482

2 files changed

Lines changed: 132 additions & 1 deletion

File tree

‎src/skillspector/providers/_agent_cli.py‎

Lines changed: 40 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@
5353
import subprocess
5454
import tempfile
5555
import threading
56+
import time
5657
from collections.abc import Callable
5758
from dataclasses import dataclass
5859
from typing import Any
@@ -632,6 +633,37 @@ def _drain_stream(stream: Any, buf: bytearray, cap: int, on_overflow: Any) -> No
632633
pass
633634

634635

636+
_CLEANUP_RETRIES = 10
637+
_CLEANUP_RETRY_DELAY_S = 0.2
638+
639+
640+
def _cleanup_temp_dir(path: str) -> None:
641+
"""Best-effort removal of the per-invocation temp cwd.
642+
643+
On Windows a directory cannot be deleted while any process has it as its
644+
working directory or holds a handle inside it. The agent CLI's process
645+
tree can outlive the direct child by a beat (a ``.cmd`` shim's node child,
646+
or a killed-but-not-reaped child on the timeout path), so the first
647+
attempts may fail transiently with ``WinError 32``. Retry briefly, then
648+
leak the directory with a warning rather than raise: by the time cleanup
649+
runs the model's response is already in hand, and a temp-dir leak must
650+
never fail the batch (#315).
651+
"""
652+
for _ in range(_CLEANUP_RETRIES):
653+
try:
654+
shutil.rmtree(path)
655+
return
656+
except OSError:
657+
time.sleep(_CLEANUP_RETRY_DELAY_S)
658+
shutil.rmtree(path, ignore_errors=True)
659+
if os.path.isdir(path):
660+
logger.warning(
661+
"Could not remove temp dir %s (handle still held by the CLI "
662+
"process tree?); leaking it rather than failing the batch",
663+
path,
664+
)
665+
666+
635667
def _run_bounded(
636668
proc: subprocess.Popen, prompt_bytes: bytes, timeout: float
637669
) -> tuple[int | None, bytes, bytes, bool]:
@@ -767,7 +799,12 @@ def run_agent_cli(
767799
child_env = _scrub_env()
768800

769801
# -- Run in a temporary directory (no CWD access) -------------------------
770-
with tempfile.TemporaryDirectory(prefix="skillspector_cli_") as tmp_cwd:
802+
# mkdtemp + explicit best-effort cleanup instead of TemporaryDirectory:
803+
# the context manager's rmtree-at-__exit__ raises on Windows while the
804+
# CLI's process tree still holds the directory as its cwd, turning an
805+
# already-successful call into a batch failure (#315).
806+
tmp_cwd = tempfile.mkdtemp(prefix="skillspector_cli_")
807+
try:
771808
logger.debug(
772809
"Running %s argv=%r cwd=%s timeout=%ss",
773810
binary_name,
@@ -792,6 +829,8 @@ def run_agent_cli(
792829
# CLI cannot exhaust memory before the cap is enforced (a chatty child
793830
# could otherwise buffer unbounded output until the timeout).
794831
returncode, stdout_raw, stderr_raw, overflow = _run_bounded(proc, prompt_bytes, timeout)
832+
finally:
833+
_cleanup_temp_dir(tmp_cwd)
795834

796835
# -- Fail-closed checks ---------------------------------------------------
797836
if overflow:

‎tests/unit/test_agent_cli.py‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -733,3 +733,95 @@ def test_is_available_false_even_when_binary_present(
733733
ok, reason = _agent_cli.is_available("agy")
734734
assert ok is False
735735
assert "disabled" in (reason or "")
736+
737+
738+
# ---------------------------------------------------------------------------
739+
# _cleanup_temp_dir — Windows-safe temp cwd removal (#315)
740+
# ---------------------------------------------------------------------------
741+
742+
743+
class TestCleanupTempDir:
744+
"""Cleanup failure must never outrank a successful CLI response (#315).
745+
746+
On Windows the CLI's process tree can hold the temp cwd (a ``.cmd``
747+
shim's node child, or a killed child on the timeout path), so rmtree can
748+
fail transiently — or persistently — after the model already answered.
749+
"""
750+
751+
def test_removes_directory(self, tmp_path) -> None:
752+
target = tmp_path / "cli_cwd"
753+
target.mkdir()
754+
(target / "scratch.txt").write_text("x")
755+
_agent_cli._cleanup_temp_dir(str(target))
756+
assert not target.exists()
757+
758+
def test_retries_transient_failure_then_succeeds(
759+
self, tmp_path, monkeypatch: pytest.MonkeyPatch
760+
) -> None:
761+
target = tmp_path / "cli_cwd"
762+
target.mkdir()
763+
real_rmtree = _agent_cli.shutil.rmtree
764+
calls = {"n": 0}
765+
766+
def flaky_rmtree(path, ignore_errors=False):
767+
calls["n"] += 1
768+
if calls["n"] < 3:
769+
raise OSError(32, "held by another process")
770+
return real_rmtree(path, ignore_errors=ignore_errors)
771+
772+
monkeypatch.setattr(_agent_cli.shutil, "rmtree", flaky_rmtree)
773+
monkeypatch.setattr(_agent_cli.time, "sleep", lambda _s: None)
774+
_agent_cli._cleanup_temp_dir(str(target))
775+
assert not target.exists()
776+
assert calls["n"] == 3
777+
778+
def test_never_raises_when_removal_keeps_failing(
779+
self, tmp_path, monkeypatch: pytest.MonkeyPatch
780+
) -> None:
781+
target = tmp_path / "cli_cwd"
782+
target.mkdir()
783+
784+
def stuck_rmtree(path, ignore_errors=False):
785+
if not ignore_errors:
786+
raise OSError(32, "held by another process")
787+
788+
monkeypatch.setattr(_agent_cli.shutil, "rmtree", stuck_rmtree)
789+
monkeypatch.setattr(_agent_cli.time, "sleep", lambda _s: None)
790+
# Must not raise; the directory is leaked deliberately.
791+
_agent_cli._cleanup_temp_dir(str(target))
792+
assert target.exists()
793+
794+
@pytest.mark.skipif(sys.platform != "win32", reason="Windows handle semantics")
795+
def test_open_handle_does_not_raise_on_windows(
796+
self, tmp_path, monkeypatch: pytest.MonkeyPatch
797+
) -> None:
798+
"""A real open handle inside the dir — the exact WinError 32 case."""
799+
target = tmp_path / "cli_cwd"
800+
target.mkdir()
801+
monkeypatch.setattr(_agent_cli.time, "sleep", lambda _s: None)
802+
held = open(target / "held.txt", "w") # noqa: SIM115 — handle held on purpose
803+
try:
804+
_agent_cli._cleanup_temp_dir(str(target)) # must not raise
805+
finally:
806+
held.close()
807+
_agent_cli._cleanup_temp_dir(str(target))
808+
assert not target.exists()
809+
810+
811+
@patch("skillspector.providers._agent_cli.find_binary", return_value=CLAUDE_BINARY)
812+
@patch("skillspector.providers._agent_cli.subprocess.Popen")
813+
class TestRunAgentCLISurvivesCleanupFailure:
814+
def test_response_returned_when_temp_dir_cannot_be_removed(
815+
self, mock_popen: MagicMock, _mock_binary: MagicMock, monkeypatch: pytest.MonkeyPatch
816+
) -> None:
817+
"""Regression for #315: rmtree failure must not fail the batch."""
818+
mock_popen.return_value = _make_ok_process(_GOOD_CLAUDE_OUTPUT.encode())
819+
820+
def stuck_rmtree(path, ignore_errors=False):
821+
if not ignore_errors:
822+
raise OSError(32, "held by another process")
823+
824+
monkeypatch.setattr(_agent_cli.shutil, "rmtree", stuck_rmtree)
825+
monkeypatch.setattr(_agent_cli.time, "sleep", lambda _s: None)
826+
result = run_agent_cli("claude", PROMPT, model=MODEL)
827+
assert result # the model's answer survives the cleanup failure

0 commit comments

Comments
 (0)