|
6 | 6 | import json |
7 | 7 | import os |
8 | 8 | import shutil |
| 9 | +import threading |
| 10 | +from collections.abc import Iterator |
| 11 | +from contextlib import contextmanager |
9 | 12 | from datetime import UTC, datetime |
10 | 13 | from pathlib import Path |
11 | 14 | from typing import Any |
|
36 | 39 | ] |
37 | 40 |
|
38 | 41 |
|
| 42 | +@contextmanager |
| 43 | +def _tier_download_lock(tier_dir: Path) -> Iterator[None]: |
| 44 | + """Serialize checkpoint swaps for one tier across threads and processes.""" |
| 45 | + tier_dir.mkdir(parents=True, exist_ok=True) |
| 46 | + lock_path = tier_dir / ".download.lock" |
| 47 | + with open(lock_path, "a+", encoding="utf-8") as lock_file: |
| 48 | + try: |
| 49 | + import fcntl |
| 50 | + |
| 51 | + fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX) |
| 52 | + except ImportError: |
| 53 | + pass |
| 54 | + try: |
| 55 | + yield |
| 56 | + finally: |
| 57 | + try: |
| 58 | + import fcntl |
| 59 | + |
| 60 | + fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN) |
| 61 | + except ImportError: |
| 62 | + pass |
| 63 | + |
| 64 | + |
39 | 65 | def _config_digest(path: Path) -> str | None: |
40 | 66 | config = path / "config.json" |
41 | 67 | if not config.is_file(): |
@@ -109,7 +135,11 @@ def write_manifest(self, manifest: ModelManifest) -> None: |
109 | 135 | def is_valid_checkpoint(self, path: Path) -> bool: |
110 | 136 | if not path.is_dir() or not (path / "config.json").is_file(): |
111 | 137 | return False |
112 | | - return any((path / name).is_file() for name in _WEIGHT_FILES) |
| 138 | + if any((path / name).is_file() for name in _WEIGHT_FILES): |
| 139 | + return True |
| 140 | + if (path / "model.safetensors.index.json").is_file(): |
| 141 | + return any(path.glob("*.safetensors")) |
| 142 | + return any(path.glob("model-*-of-*.safetensors")) |
113 | 143 |
|
114 | 144 | def _installed_checkpoint(self, tier: str) -> Path | None: |
115 | 145 | """Return cached checkpoint when manifest + weights exist (revision may be stale).""" |
@@ -166,57 +196,67 @@ def ensure_tier( |
166 | 196 | from huggingface_hub import snapshot_download |
167 | 197 |
|
168 | 198 | dest = self.tier_dir(tier) / "checkpoint" |
169 | | - temp_dest = self.tier_dir(tier) / f".checkpoint-download-{os.getpid()}" |
170 | | - backup = self.tier_dir(tier) / f".checkpoint-{os.getpid()}.bak" |
| 199 | + staging_id = f"{os.getpid()}-{threading.get_ident()}" |
| 200 | + temp_dest = self.tier_dir(tier) / f".checkpoint-download-{staging_id}" |
| 201 | + backup = self.tier_dir(tier) / f".checkpoint-{staging_id}.bak" |
171 | 202 |
|
172 | | - if temp_dest.exists(): |
173 | | - shutil.rmtree(temp_dest) |
174 | | - temp_dest.mkdir(parents=True, exist_ok=True) |
175 | | - |
176 | | - moved_to_backup = False |
177 | | - try: |
178 | | - local_dir = snapshot_download( |
179 | | - repo_id=tier_entry.repo_id, |
180 | | - revision=tier_entry.revision, |
181 | | - local_dir=str(temp_dest), |
182 | | - allow_patterns=_SNAPSHOT_ALLOW_PATTERNS, |
183 | | - ) |
184 | | - ckpt = Path(local_dir) |
185 | | - if not self.is_valid_checkpoint(ckpt): |
186 | | - msg = ( |
187 | | - f"Downloaded model at {ckpt} is missing config.json or weight files. " |
188 | | - f"Check repo {tier_entry.repo_id!r} on Hugging Face." |
| 203 | + with _tier_download_lock(self.tier_dir(tier)): |
| 204 | + if temp_dest.exists(): |
| 205 | + shutil.rmtree(temp_dest) |
| 206 | + temp_dest.mkdir(parents=True, exist_ok=True) |
| 207 | + |
| 208 | + moved_to_backup = False |
| 209 | + manifest_written = False |
| 210 | + previous_manifest = self.read_manifest(tier) |
| 211 | + try: |
| 212 | + local_dir = snapshot_download( |
| 213 | + repo_id=tier_entry.repo_id, |
| 214 | + revision=tier_entry.revision, |
| 215 | + local_dir=str(temp_dest), |
| 216 | + allow_patterns=_SNAPSHOT_ALLOW_PATTERNS, |
| 217 | + ) |
| 218 | + ckpt = Path(local_dir) |
| 219 | + if not self.is_valid_checkpoint(ckpt): |
| 220 | + msg = ( |
| 221 | + f"Downloaded model at {ckpt} is missing config.json or weight files. " |
| 222 | + f"Check repo {tier_entry.repo_id!r} on Hugging Face." |
| 223 | + ) |
| 224 | + raise ConfigError(msg) |
| 225 | + |
| 226 | + manifest = ModelManifest( |
| 227 | + tier=tier, |
| 228 | + repo_id=tier_entry.repo_id, |
| 229 | + revision=tier_entry.revision, |
| 230 | + path=str(dest), |
| 231 | + config_digest=_config_digest(ckpt), |
189 | 232 | ) |
190 | | - raise ConfigError(msg) |
191 | | - |
192 | | - manifest = ModelManifest( |
193 | | - tier=tier, |
194 | | - repo_id=tier_entry.repo_id, |
195 | | - revision=tier_entry.revision, |
196 | | - path=str(dest), |
197 | | - config_digest=_config_digest(ckpt), |
198 | | - ) |
199 | 233 |
|
200 | | - if backup.exists(): |
201 | | - shutil.rmtree(backup) |
202 | | - if dest.exists(): |
203 | | - os.replace(dest, backup) |
204 | | - moved_to_backup = True |
205 | | - os.replace(ckpt, dest) |
206 | | - self.write_manifest(manifest) |
207 | | - if backup.exists(): |
208 | | - shutil.rmtree(backup) |
209 | | - return dest |
210 | | - except Exception: |
211 | | - # Roll back to the last good checkpoint so a failed swap never |
212 | | - # strands the tier without a model (manifest still matches it). |
213 | | - if moved_to_backup and backup.exists(): |
| 234 | + if backup.exists(): |
| 235 | + shutil.rmtree(backup) |
214 | 236 | if dest.exists(): |
215 | | - shutil.rmtree(dest) |
216 | | - os.replace(backup, dest) |
217 | | - if temp_dest.exists(): |
218 | | - shutil.rmtree(temp_dest) |
219 | | - raise |
| 237 | + os.replace(dest, backup) |
| 238 | + moved_to_backup = True |
| 239 | + os.replace(ckpt, dest) |
| 240 | + self.write_manifest(manifest) |
| 241 | + manifest_written = True |
| 242 | + if backup.exists(): |
| 243 | + shutil.rmtree(backup) |
| 244 | + return dest |
| 245 | + except Exception: |
| 246 | + # Roll back to the last good checkpoint so a failed swap never |
| 247 | + # strands the tier without a model (manifest still matches it). |
| 248 | + if moved_to_backup and backup.exists(): |
| 249 | + if dest.exists(): |
| 250 | + shutil.rmtree(dest) |
| 251 | + os.replace(backup, dest) |
| 252 | + if manifest_written: |
| 253 | + if previous_manifest is not None: |
| 254 | + self.write_manifest(previous_manifest) |
| 255 | + elif self.manifest_path(tier).is_file(): |
| 256 | + self.manifest_path(tier).unlink() |
| 257 | + if temp_dest.exists(): |
| 258 | + shutil.rmtree(temp_dest) |
| 259 | + raise |
220 | 260 |
|
221 | 261 | def list_status(self) -> list[dict[str, Any]]: |
222 | 262 | """Status for every catalog tier (installed / upgrade available).""" |
|
0 commit comments