Skip to content

Commit 7d36728

Browse files
authored
fix: load ONNX external data from the huggingface_hub>=1.32 cache (#757)
1 parent 2f2a8bf commit 7d36728

18 files changed

Lines changed: 163 additions & 6 deletions

‎fastembed/common/model_management.py‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
from loguru import logger
2525
from tqdm import tqdm
2626
from fastembed.common.model_description import BaseModelDescription
27+
from fastembed.common.onnx_external_data import link_external_data
2728

2829
T = TypeVar("T", bound=BaseModelDescription)
2930

@@ -584,6 +585,19 @@ def retrieve_model_gcs(
584585

585586
return model_dir
586587

588+
@staticmethod
589+
def _link_onnx_external_data(model: BaseModelDescription, model_dir: Path) -> Path:
590+
"""Links the external data of `model` right after its download, see link_external_data.
591+
592+
Loading the model links it as well, but linking it here also covers lazy_load=True, e.g.
593+
in a Docker build step, where links made by a later step would copy the files into a new
594+
image layer. A failure, e.g. in a read-only cache, is reported when the model is loaded.
595+
"""
596+
if model.model_file.endswith(".onnx"):
597+
with contextlib.suppress(OSError):
598+
link_external_data(model_dir, model.model_file, model.additional_files)
599+
return model_dir
600+
587601
@classmethod
588602
def download_model(cls, model: T, cache_dir: str, retries: int = 3, **kwargs: Any) -> Path:
589603
"""
@@ -645,7 +659,7 @@ def download_model(cls, model: T, cache_dir: str, retries: int = 3, **kwargs: An
645659
if (resolved_path / model.model_file).exists() and all(
646660
(resolved_path / file).exists() for file in extra_patterns
647661
):
648-
return resolved_path
662+
return cls._link_onnx_external_data(model, resolved_path)
649663
except CorruptedCacheError:
650664
force_download = True
651665
except Exception:
@@ -667,14 +681,15 @@ def download_model(cls, model: T, cache_dir: str, retries: int = 3, **kwargs: An
667681
attempt_kwargs = {**kwargs, "force_download": True} if force_download else kwargs
668682
force_download = False
669683
try:
670-
return Path(
684+
model_dir = Path(
671685
cls.download_files_from_huggingface(
672686
hf_source,
673687
cache_dir=cache_dir,
674688
extra_patterns=extra_patterns,
675689
**attempt_kwargs,
676690
)
677691
)
692+
return cls._link_onnx_external_data(model, model_dir)
678693
except _HF_DOWNLOAD_ERRORS as e:
679694
logger.error(
680695
f"Could not download model from HuggingFace: {e} "
Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
"""Loading ONNX models with external data from a huggingface_hub cache.
2+
3+
onnxruntime>=1.24 refuses external data that, once symlinks are resolved, is located outside of
4+
the directory of the model file, and of the directory the model file resolves to (a fallback of
5+
1.24.2, pyproject.toml excludes 1.24.0 and 1.24.1). Since huggingface_hub 1.32, a snapshot is made
6+
of symlinks into a blob store shared by the whole cache and sharded by hash, where a model and its
7+
data may resolve into different directories. Then the model and its data are hardlinked into
8+
`ONNX_SNAPSHOTS_DIR` of the repo cache, laid out as in the snapshot, which takes no extra space.
9+
"""
10+
11+
import contextlib
12+
import os
13+
import shutil
14+
from pathlib import Path
15+
16+
# Next to `snapshots` in the cache of a huggingface_hub repo: `onnx_snapshots/<revision>/...`
17+
ONNX_SNAPSHOTS_DIR = "onnx_snapshots"
18+
19+
20+
def link_external_data(model_dir: Path, model_file: str, additional_files: list[str]) -> Path:
21+
"""Returns a path of `model_file` from which onnxruntime can load its external data.
22+
23+
Args:
24+
model_dir (Path): The directory with the model files, e.g. a huggingface_hub snapshot.
25+
model_file (str): The path of the ONNX file, relative to `model_dir`.
26+
additional_files (list[str]): Other files of the model, relative to `model_dir`,
27+
among which its external data.
28+
29+
Returns:
30+
Path: The ONNX file to load, `model_dir / model_file` unless it had to be linked.
31+
32+
Raises:
33+
OSError: If the files had to be linked, but couldn't be, e.g. in a read-only cache or
34+
on a filesystem without hardlinks.
35+
"""
36+
model_path = model_dir / model_file
37+
snapshots_dir = model_dir.parent
38+
repo_dir = snapshots_dir.parent
39+
if snapshots_dir.name != "snapshots" or not repo_dir.name.startswith("models--"):
40+
return model_path # not a huggingface_hub cache, there is no place of ours to link into
41+
if not model_path.exists():
42+
return model_path # onnxruntime reports it more clearly than a failed hardlink would
43+
44+
# external data is located relative to the model file, so it can't be anywhere else
45+
data_paths = [
46+
path
47+
for path in (model_dir / file for file in additional_files)
48+
if path.parent.is_relative_to(model_path.parent) and path.exists()
49+
]
50+
# onnxruntime accepts data within these, as in the caches of older huggingface_hub versions
51+
real_model_dirs = (
52+
os.path.realpath(model_path.parent),
53+
os.path.dirname(os.path.realpath(model_path)),
54+
)
55+
if all(
56+
any(Path(os.path.realpath(path)).is_relative_to(d) for d in real_model_dirs)
57+
for path in data_paths
58+
):
59+
return model_path
60+
61+
links_dir = repo_dir / ONNX_SNAPSHOTS_DIR
62+
for path in (model_path, *data_paths):
63+
_link_file(path, links_dir / model_dir.name / path.relative_to(model_dir))
64+
65+
# the links keep the blobs on disk, so drop those of revisions deleted from the cache
66+
for revision_dir in links_dir.iterdir():
67+
if not (snapshots_dir / revision_dir.name).is_dir():
68+
shutil.rmtree(revision_dir, ignore_errors=True)
69+
70+
return links_dir / model_dir.name / model_file
71+
72+
73+
def _link_file(source: Path, link: Path) -> None:
74+
"""Makes `link` a hardlink to the file `source` resolves to, unless it already is one."""
75+
target = os.path.realpath(source)
76+
try:
77+
if os.path.samefile(link, target):
78+
return
79+
# a link to a blob downloaded again since, or a copy of the cache without its hardlinks
80+
link.unlink()
81+
except FileNotFoundError:
82+
pass
83+
except OSError:
84+
# e.g. such a copy on a read-only filesystem, which works all the same
85+
if os.path.getsize(link) != os.path.getsize(target):
86+
raise
87+
return
88+
link.parent.mkdir(parents=True, exist_ok=True)
89+
# os.link is atomic, so if the link exists, another process loading the model just made it
90+
with contextlib.suppress(FileExistsError):
91+
os.link(target, link)

‎fastembed/common/onnx_model.py‎

Lines changed: 26 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,11 @@
66
import numpy as np
77
import onnxruntime as ort
88

9+
from loguru import logger
910
from numpy.typing import NDArray
1011
from tokenizers import Tokenizer
1112

13+
from fastembed.common.onnx_external_data import ONNX_SNAPSHOTS_DIR, link_external_data
1214
from fastembed.common.types import OnnxProvider, NumpyArray, Device
1315
from fastembed.parallel_processor import Worker
1416

@@ -78,8 +80,8 @@ def _load_onnx_model(
7880
cuda: bool | Device = Device.AUTO,
7981
device_id: int | None = None,
8082
extra_session_options: dict[str, Any] | None = None,
83+
additional_files: list[str] | None = None,
8184
) -> None:
82-
model_path = model_dir / model_file
8385
# List of Execution Providers: https://onnxruntime.ai/docs/execution-providers
8486
available_providers = ort.get_available_providers()
8587
cuda_available = "CUDAExecutionProvider" in available_providers
@@ -124,9 +126,29 @@ def _load_onnx_model(
124126
if extra_session_options is not None:
125127
self.add_extra_session_options(so, extra_session_options)
126128

127-
self.model = ort.InferenceSession(
128-
str(model_path), providers=onnx_providers, sess_options=so
129-
)
129+
model_path = model_dir / model_file
130+
link_error: OSError | None = None
131+
try:
132+
model_path = link_external_data(model_dir, model_file, additional_files or [])
133+
except OSError as e:
134+
# e.g. a read-only cache, which onnxruntime<1.24 loads from all the same
135+
link_error = e
136+
137+
try:
138+
self.model = ort.InferenceSession(
139+
str(model_path), providers=onnx_providers, sess_options=so
140+
)
141+
except Exception:
142+
if link_error is not None:
143+
logger.warning(
144+
f"Could not link the files of {model_path} into {ONNX_SNAPSHOTS_DIR}: "
145+
f"{link_error}. onnxruntime>=1.24 refuses external data that resolves "
146+
"outside of the model directory, as in a huggingface_hub>=1.32 cache. "
147+
"Download the model with fastembed into a writable cache_dir on a "
148+
"filesystem with hardlinks, or delete it from the cache and download it "
149+
"again with HF_HUB_DISABLE_SHARED_BLOBS=1."
150+
)
151+
raise
130152
if "CUDAExecutionProvider" in requested_provider_names:
131153
assert self.model is not None
132154
current_providers = self.model.get_providers()

‎fastembed/image/onnx_embedding.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,7 @@ def load_onnx_model(self) -> None:
137137
cuda=self.cuda,
138138
device_id=self.device_id,
139139
extra_session_options=self._extra_session_options,
140+
additional_files=self.model_description.additional_files,
140141
)
141142

142143
@classmethod

‎fastembed/image/onnx_image_model.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ def _load_onnx_model(
5858
cuda: bool | Device = Device.AUTO,
5959
device_id: int | None = None,
6060
extra_session_options: dict[str, Any] | None = None,
61+
additional_files: list[str] | None = None,
6162
) -> None:
6263
super()._load_onnx_model(
6364
model_dir=model_dir,
@@ -67,6 +68,7 @@ def _load_onnx_model(
6768
cuda=cuda,
6869
device_id=device_id,
6970
extra_session_options=extra_session_options,
71+
additional_files=additional_files,
7072
)
7173
self.processor = load_preprocessor(model_dir=model_dir)
7274

‎fastembed/late_interaction/colbert.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -221,6 +221,7 @@ def load_onnx_model(self) -> None:
221221
cuda=self.cuda,
222222
device_id=self.device_id,
223223
extra_session_options=self._extra_session_options,
224+
additional_files=self.model_description.additional_files,
224225
)
225226

226227
def _load_tokenizer(self, model_dir: Path) -> None:

‎fastembed/late_interaction_multimodal/colmodernvbert.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,7 @@ def load_onnx_model(self) -> None:
133133
cuda=self.cuda,
134134
device_id=self.device_id,
135135
extra_session_options=self._extra_session_options,
136+
additional_files=self.model_description.additional_files,
136137
)
137138

138139
# Load image processing configuration

‎fastembed/late_interaction_multimodal/colpali.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,7 @@ def load_onnx_model(self) -> None:
128128
cuda=self.cuda,
129129
device_id=self.device_id,
130130
extra_session_options=self._extra_session_options,
131+
additional_files=self.model_description.additional_files,
131132
)
132133

133134
def _post_process_onnx_image_output(

‎fastembed/late_interaction_multimodal/onnx_multimodal_model.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ def _load_onnx_model(
6565
cuda: bool | Device = Device.AUTO,
6666
device_id: int | None = None,
6767
extra_session_options: dict[str, Any] | None = None,
68+
additional_files: list[str] | None = None,
6869
) -> None:
6970
super()._load_onnx_model(
7071
model_dir=model_dir,
@@ -74,6 +75,7 @@ def _load_onnx_model(
7475
cuda=cuda,
7576
device_id=device_id,
7677
extra_session_options=extra_session_options,
78+
additional_files=additional_files,
7779
)
7880
self._ensure_tokenizer()
7981
assert self.tokenizer is not None

‎fastembed/rerank/cross_encoder/onnx_text_cross_encoder.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -171,6 +171,7 @@ def load_onnx_model(self) -> None:
171171
cuda=self.cuda,
172172
device_id=self.device_id,
173173
extra_session_options=self._extra_session_options,
174+
additional_files=self.model_description.additional_files,
174175
)
175176

176177
def rerank(

0 commit comments

Comments
 (0)