Skip to content

Commit 25580e2

Browse files
committed
Restore backup on failed checkpoint swap
1 parent f22eb23 commit 25580e2

2 files changed

Lines changed: 60 additions & 2 deletions

File tree

‎sdk/src/unplug/ml/store.py‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@
2222

2323
_WEIGHT_FILES = ("model.safetensors", "pytorch_model.bin")
2424

25+
# Downloads are safetensors-only by policy (no pickle .bin fetches); .bin
26+
# stays in _WEIGHT_FILES so pre-existing local checkpoints still validate.
2527
_SNAPSHOT_ALLOW_PATTERNS = [
2628
"*.json",
2729
"*.safetensors",
@@ -165,12 +167,13 @@ def ensure_tier(
165167

166168
dest = self.tier_dir(tier) / "checkpoint"
167169
temp_dest = self.tier_dir(tier) / f".checkpoint-download-{os.getpid()}"
168-
backup = self.tier_dir(tier) / ".checkpoint.bak"
170+
backup = self.tier_dir(tier) / f".checkpoint-{os.getpid()}.bak"
169171

170172
if temp_dest.exists():
171173
shutil.rmtree(temp_dest)
172174
temp_dest.mkdir(parents=True, exist_ok=True)
173175

176+
moved_to_backup = False
174177
try:
175178
local_dir = snapshot_download(
176179
repo_id=tier_entry.repo_id,
@@ -198,12 +201,19 @@ def ensure_tier(
198201
shutil.rmtree(backup)
199202
if dest.exists():
200203
os.replace(dest, backup)
204+
moved_to_backup = True
201205
os.replace(ckpt, dest)
202-
self.write_manifest(manifest.model_copy(update={"path": str(dest)}))
206+
self.write_manifest(manifest)
203207
if backup.exists():
204208
shutil.rmtree(backup)
205209
return dest
206210
except Exception:
211+
# Roll back to the last good checkpoint so a failed swap never
212+
# strands the tier without a model (manifest still matches it).
213+
if moved_to_backup and backup.exists():
214+
if dest.exists():
215+
shutil.rmtree(dest)
216+
os.replace(backup, dest)
207217
if temp_dest.exists():
208218
shutil.rmtree(temp_dest)
209219
raise

‎sdk/tests/unit/ml/test_model_store.py‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -280,6 +280,54 @@ def boom(**kwargs: object) -> str:
280280
assert store.resolve_local_path("tiny") == ckpt
281281

282282

283+
def test_failed_swap_restores_backup_checkpoint(
284+
tmp_path: Path,
285+
monkeypatch: pytest.MonkeyPatch,
286+
) -> None:
287+
"""Failure after the old checkpoint moved to backup must roll it back."""
288+
store = ModelStore(cache_root=tmp_path)
289+
ckpt = tmp_path / "tiny" / "checkpoint"
290+
_write_checkpoint(ckpt, config='{"model": "keep"}')
291+
digest = _config_digest(ckpt)
292+
store.write_manifest(
293+
ModelManifest(
294+
tier="tiny",
295+
repo_id="Unplug-AI/test",
296+
revision=load_catalog().tiers["tiny"].revision,
297+
path=str(ckpt),
298+
config_digest=digest,
299+
)
300+
)
301+
302+
def fake_snapshot_download(**kwargs: object) -> str:
303+
local_dir = Path(str(kwargs["local_dir"]))
304+
_write_checkpoint(local_dir, config='{"model": "new"}')
305+
return str(local_dir)
306+
307+
hf = ModuleType("huggingface_hub")
308+
hf.snapshot_download = fake_snapshot_download # type: ignore[attr-defined]
309+
monkeypatch.setitem(sys.modules, "huggingface_hub", hf)
310+
monkeypatch.setattr(
311+
"unplug.ml.store.require_huggingface_hub",
312+
lambda: None,
313+
)
314+
315+
def broken_write(manifest: ModelManifest) -> None:
316+
msg = "disk full"
317+
raise OSError(msg)
318+
319+
monkeypatch.setattr(store, "write_manifest", broken_write)
320+
321+
with pytest.raises(OSError, match="disk full"):
322+
store.ensure_tier("tiny", force=True)
323+
324+
# Old checkpoint restored at the original path; manifest still matches it.
325+
assert (ckpt / "config.json").read_text(encoding="utf-8") == '{"model": "keep"}'
326+
assert store.resolve_local_path("tiny") == ckpt
327+
leftovers = [p.name for p in (tmp_path / "tiny").iterdir() if p.name != "checkpoint"]
328+
assert leftovers == ["manifest.json"]
329+
330+
283331
def test_invalid_unplug_model_path_logs_warning(
284332
tmp_path: Path,
285333
monkeypatch: pytest.MonkeyPatch,

0 commit comments

Comments
 (0)