Skip to content

Commit 65e9ece

Browse files
MaxFreedomPollardteknium1
authored andcommitted
fix(curator): restore the real skills tree when a rollback extract dies part-way
shutil.move() moves into an existing destination directory instead of replacing it. When the snapshot extract failed after creating some of its output, the recovery path moved each staged entry onto a path the extract had already created, burying the user's own skill one level deeper (skills/alpha/alpha/) and leaving the snapshot's partial content in its place. rollback() then returned "snapshot extract failed (state restored)". Clear the failed extract's output before moving the staged copies back: entries the original tree never had are dropped, and each staged entry's destination is removed before the move. When an entry still cannot be restored, keep the staging dir and name the entries in the message rather than reporting a restore that did not happen.
1 parent 31032b4 commit 65e9ece

2 files changed

Lines changed: 115 additions & 8 deletions

File tree

‎agent/curator_backup.py‎

Lines changed: 49 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -541,6 +541,33 @@ def _restore_cron_skill_links(snapshot_dir: Path) -> Dict[str, Any]:
541541

542542

543543

544+
def _unstage(moved: List[Tuple[Path, Path]]) -> List[str]:
545+
"""Move staged entries back to their original paths.
546+
547+
``shutil.move`` moves *into* an existing destination directory rather than
548+
replacing it, so a partially-completed extract leaves debris that would
549+
otherwise bury the user's real skill one level deeper
550+
(``skills/foo/foo/``) while the tree still looks populated. Clear whatever
551+
the failed extract created at each original path first. The staged copy is
552+
authoritative, and the pre-rollback safety snapshot is the undo handle for
553+
the extract's own output.
554+
555+
Returns the names that could not be restored, so the caller can report an
556+
incomplete recovery instead of claiming the state was restored.
557+
"""
558+
failed: List[str] = []
559+
for orig, dest in moved:
560+
try:
561+
if orig.is_dir() and not orig.is_symlink():
562+
shutil.rmtree(orig)
563+
elif orig.exists() or orig.is_symlink():
564+
orig.unlink()
565+
shutil.move(str(dest), str(orig))
566+
except OSError:
567+
failed.append(orig.name)
568+
return failed
569+
570+
544571
def rollback(backup_id: Optional[str] = None) -> Tuple[bool, str, Optional[Path]]:
545572
"""Restore ``~/.hermes/skills/`` from a snapshot.
546573
@@ -609,11 +636,7 @@ def rollback(backup_id: Optional[str] = None) -> Tuple[bool, str, Optional[Path]
609636
moved.append((entry, dest))
610637
except OSError as e:
611638
# Best-effort rollback of the move
612-
for orig, dest in moved:
613-
try:
614-
shutil.move(str(dest), str(orig))
615-
except OSError:
616-
pass
639+
_unstage(moved)
617640
try:
618641
shutil.rmtree(staged, ignore_errors=True)
619642
except OSError:
@@ -638,12 +661,30 @@ def rollback(backup_id: Optional[str] = None) -> Tuple[bool, str, Optional[Path]
638661
# Python < 3.12 — no filter kwarg
639662
tf.extractall(str(skills))
640663
except (OSError, tarfile.TarError) as e:
641-
# Best-effort recover: move staged contents back
642-
for orig, dest in moved:
664+
# Best-effort recover. A partial extract can leave entries the
665+
# original tree never had, so drop those first, otherwise the
666+
# "restored" tree is the user's skills plus a slice of the snapshot.
667+
staged_names = {orig.name for orig, _ in moved}
668+
for entry in list(skills.iterdir()):
669+
if entry.name in _EXCLUDE_TOP_LEVEL or entry.name in staged_names:
670+
continue
643671
try:
644-
shutil.move(str(dest), str(orig))
672+
if entry.is_dir() and not entry.is_symlink():
673+
shutil.rmtree(entry)
674+
else:
675+
entry.unlink()
645676
except OSError:
646677
pass
678+
unrestored = _unstage(moved)
679+
if unrestored:
680+
# Do not claim a clean restore we did not achieve, and keep the
681+
# staging dir so the entries can be recovered by hand.
682+
return (
683+
False,
684+
f"snapshot extract failed: {e} - could not restore "
685+
f"{', '.join(sorted(unrestored))}; staged copies kept at {staged}",
686+
None,
687+
)
647688
try:
648689
shutil.rmtree(staged, ignore_errors=True)
649690
except OSError:

‎tests/agent/test_curator_backup.py‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -392,3 +392,69 @@ def _three_ordered_snapshots(cb, skills, monkeypatch):
392392

393393

394394

395+
# ---------------------------------------------------------------------------
396+
# A failed extract must leave the skills tree exactly as it was found
397+
# ---------------------------------------------------------------------------
398+
399+
def test_rollback_recovers_cleanly_from_a_partial_extract(backup_env, monkeypatch):
400+
"""An extract that dies part-way must restore the original tree exactly.
401+
402+
``shutil.move`` moves *into* an existing directory rather than replacing
403+
it, so debris from a half-finished extract buried the user's own skill one
404+
level deeper (``skills/alpha/alpha/``) while rollback still reported
405+
"state restored".
406+
"""
407+
cb = backup_env["cb"]
408+
skills = backup_env["skills"]
409+
410+
_write_skill(skills, "alpha", body="snapshot copy")
411+
assert cb.snapshot_skills(reason="before") is not None
412+
413+
# Diverge from the snapshot so a real restore would be observable.
414+
_write_skill(skills, "alpha", body="current copy")
415+
_write_skill(skills, "beta", body="current only")
416+
417+
real_open = tarfile.open
418+
419+
class _DiesMidExtract:
420+
"""Writes part of the archive, then fails like a full disk would."""
421+
422+
def __init__(self, inner):
423+
self._inner = inner
424+
425+
def getmembers(self):
426+
return self._inner.getmembers()
427+
428+
def extractall(self, path, *args, **kwargs):
429+
partial = Path(path) / "alpha"
430+
partial.mkdir(parents=True, exist_ok=True)
431+
(partial / "SKILL.md").write_text("half written", encoding="utf-8")
432+
(Path(path) / "gamma").mkdir(parents=True, exist_ok=True)
433+
raise OSError(28, "No space left on device")
434+
435+
def __enter__(self):
436+
return self
437+
438+
def __exit__(self, *exc):
439+
self._inner.close()
440+
return False
441+
442+
def _open(name, mode="r", *args, **kwargs):
443+
# Only the rollback's read is intercepted; the pre-rollback safety
444+
# snapshot opens for write and must keep working.
445+
handle = real_open(name, mode, *args, **kwargs)
446+
return _DiesMidExtract(handle) if mode.startswith("r") else handle
447+
448+
monkeypatch.setattr(cb.tarfile, "open", _open)
449+
450+
ok, msg, _ = cb.rollback()
451+
assert not ok
452+
453+
assert not (skills / "alpha" / "alpha").exists(), \
454+
"staged skill was nested, not restored"
455+
present = sorted(
456+
p.name for p in skills.iterdir() if p.name not in cb._EXCLUDE_TOP_LEVEL
457+
)
458+
assert present == ["alpha", "beta"], f"tree not restored: {present}"
459+
assert "current copy" in (skills / "alpha" / "SKILL.md").read_text(encoding="utf-8")
460+
assert "current only" in (skills / "beta" / "SKILL.md").read_text(encoding="utf-8")

0 commit comments

Comments
 (0)