Skip to content

Commit 184e1fc

Browse files
Studio: use llama-server for embedding models the installed sentence-transformers cannot load (unslothai#13005)
* Studio: plan llama-server for embedding models sentence-transformers cannot open * Skip the safetensors fallback for models sentence-transformers cannot load * Keep a proven sentence-transformers incompatibility for the rest of the resolution * Trim comments * Record embedding preflight import gate * Trim comments in PR scope * Fix embedding preflight metadata resolution * Scope embedding config preflight --------- Co-authored-by: mahiatlinux <[email protected]>
1 parent f7d992c commit 184e1fc

5 files changed

Lines changed: 459 additions & 70 deletions

File tree

‎scripts/risky_loaders_baseline.json‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -250,6 +250,14 @@
250250
"count": 1,
251251
"reason": "Prefix-bounded to core.inference.<gallery>; the name comes from the fixed _GALLERIES table."
252252
},
253+
{
254+
"file": "studio/backend/core/rag/embeddings.py",
255+
"rule": "dynamic-import",
256+
"sink": "import_from_string",
257+
"digest": "ad319be07f7da5ad",
258+
"count": 1,
259+
"reason": "The metadata reference is accepted only with the sentence_transformers. prefix, so it can import installed package code only; non-package refs are skipped and the real loader performs the same lookup."
260+
},
253261
{
254262
"file": "studio/backend/core/systemone/laya_runtime.py",
255263
"rule": "dynamic-import",

‎studio/backend/core/rag/embeddings.py‎

Lines changed: 131 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,9 @@
2424
import re
2525
import threading
2626
from contextlib import contextmanager
27+
from contextvars import ContextVar
2728
from functools import lru_cache
28-
from typing import Callable
29+
from typing import Any, Callable
2930

3031
from utils.hardware.hardware import DeviceType, get_device
3132
from utils.transformers_dtype import dtype_kwargs
@@ -171,6 +172,38 @@ def _ambient_hf_token() -> str | None:
171172
return None
172173

173174

175+
def _repo_json(name: str, filename: str, token: str | bool | None) -> Any:
176+
"""``filename`` parsed from the local folder or Hub repo ``name``, None when the file is absent.
177+
Raises whatever reading or parsing raises."""
178+
import json
179+
180+
from utils.paths import is_local_path
181+
182+
if is_local_path(name):
183+
from pathlib import Path
184+
from utils.paths import normalize_path
185+
186+
path = Path(normalize_path(name)).expanduser() / filename
187+
if not path.is_file():
188+
return None
189+
return json.loads(path.read_text(encoding = "utf-8-sig"))
190+
from huggingface_hub import hf_hub_download
191+
from huggingface_hub.utils import EntryNotFoundError
192+
from utils.hf_cache_settings import active_hf_hub_cache
193+
194+
try:
195+
local = hf_hub_download(
196+
name,
197+
filename,
198+
token = account_hf_token(token),
199+
cache_dir = active_hf_hub_cache(),
200+
)
201+
except EntryNotFoundError:
202+
return None
203+
with open(local, encoding = "utf-8-sig") as f:
204+
return json.load(f)
205+
206+
174207
def _st_module_subdirs(name: str, token: str | None) -> tuple[str, ...]:
175208
"""The module directories a SentenceTransformer load reads weights from, taken from
176209
the repo's ``modules.json`` (each module's non-empty ``path``, e.g. ``0_Transformer``).
@@ -179,33 +212,7 @@ def _st_module_subdirs(name: str, token: str | None) -> tuple[str, ...]:
179212
failure (no modules.json, offline, malformed) so the guard never bricks the embedder.
180213
"""
181214
try:
182-
import json
183-
184-
from utils.paths import is_local_path
185-
186-
if is_local_path(name):
187-
from pathlib import Path
188-
from utils.paths import normalize_path
189-
190-
path = Path(normalize_path(name)).expanduser() / "modules.json"
191-
if not path.is_file():
192-
return ()
193-
data = json.loads(path.read_text(encoding = "utf-8-sig"))
194-
else:
195-
from huggingface_hub import hf_hub_download
196-
from huggingface_hub.utils import EntryNotFoundError
197-
from utils.hf_cache_settings import active_hf_hub_cache
198-
199-
try:
200-
local = hf_hub_download(
201-
name,
202-
"modules.json",
203-
token = account_hf_token(token or None),
204-
cache_dir = active_hf_hub_cache(),
205-
)
206-
except EntryNotFoundError:
207-
return ()
208-
data = json.loads(open(local, encoding = "utf-8-sig").read())
215+
data = _repo_json(name, "modules.json", token)
209216
subdirs = []
210217
for module in data or ():
211218
sub = str((module or {}).get("path", "")).strip().strip("/")
@@ -908,15 +915,106 @@ def _llama_server_runtime_available() -> bool:
908915
return False
909916

910917

911-
def resolved_backend_for_model(model_name: str) -> str:
918+
_ST_LOAD_PREFLIGHT_TIMEOUT_S = 5.0
919+
# Proofs last one Settings resolution: a folder can be fixed, a repo republished, or the next caller lack access.
920+
_st_unloadable_proofs: ContextVar[set[str] | None] = ContextVar(
921+
"st-unloadable-proofs", default = None
922+
)
923+
924+
925+
@contextmanager
926+
def st_load_proof_scope():
927+
"""Keep ``sentence_transformers_can_load`` proofs for the enclosed resolution. Nested scopes share one."""
928+
marker = _st_unloadable_proofs.set(set()) if _st_unloadable_proofs.get() is None else None
929+
try:
930+
yield
931+
finally:
932+
if marker is not None:
933+
_st_unloadable_proofs.reset(marker)
934+
935+
936+
def sentence_transformers_known_unloadable(model_name: str) -> bool:
937+
"""Whether a check earlier in this scope proved ``model_name`` unloadable. No network."""
938+
proofs = _st_unloadable_proofs.get()
939+
return proofs is not None and model_name in proofs
940+
941+
942+
def sentence_transformers_can_load(model_name: str, token: str | bool | None = None) -> bool:
943+
"""False when ``modules.json`` names a sentence_transformers class this install lacks (embeddinggemma-2 names
944+
6.x ``sentence_transformers.base`` modules) or ``config.json`` a ``model_type`` transformers does not know.
945+
True when either file cannot be read, so an unreachable repo keeps its plan."""
946+
if sentence_transformers_known_unloadable(model_name):
947+
return False
948+
if _st_load_preflight(model_name, token):
949+
return True
950+
proofs = _st_unloadable_proofs.get()
951+
if proofs is not None:
952+
proofs.add(model_name)
953+
return False
954+
955+
956+
def _st_load_preflight(model_name: str, token: str | bool | None = None) -> bool:
957+
try:
958+
from sentence_transformers.util import import_from_string
959+
from utils.utils import call_with_deadline
960+
961+
def _metadata():
962+
modules = _repo_json(model_name, "modules.json", token)
963+
config_files = ["config.json"] if modules is None else []
964+
for module in modules or ():
965+
ref = str((module or {}).get("type", ""))
966+
if ref.rsplit(".", 1)[-1] != "Transformer":
967+
continue
968+
path = str((module or {}).get("path", "")).strip("/")
969+
filename = f"{path}/config.json" if path else "config.json"
970+
if filename not in config_files:
971+
config_files.append(filename)
972+
return modules, {
973+
filename: _repo_json(model_name, filename, token) for filename in config_files
974+
}
975+
976+
# bounded with one deadline so nested module metadata cannot multiply the wait
977+
modules, model_configs = call_with_deadline(
978+
_metadata,
979+
_ST_LOAD_PREFLIGHT_TIMEOUT_S,
980+
name = "embed-st-load-preflight",
981+
)
982+
for module in modules or ():
983+
ref = str((module or {}).get("type", ""))
984+
# Other classes are third-party code, which _gate_st_custom_modules refuses on its own.
985+
if not ref.startswith(_ST_PACKAGE_PREFIX):
986+
continue
987+
try:
988+
# How ST's own loader resolves sentence_transformers.* refs.
989+
import_from_string(ref)
990+
except ImportError:
991+
return False
992+
for model_config in model_configs.values():
993+
model_type = (model_config or {}).get("model_type")
994+
if isinstance(model_type, str):
995+
# CONFIG_MAPPING, not CONFIG_MAPPING_NAMES, so types registered at runtime count. An auto_map does not
996+
# help an unknown type: the loader never passes trust_remote_code.
997+
from transformers import CONFIG_MAPPING
998+
if model_type not in CONFIG_MAPPING:
999+
return False
1000+
except Exception as exc: # noqa: BLE001 - offline, gated, missing or malformed: no proof either way
1001+
logger.debug("sentence-transformers load preflight for %s failed: %s", model_name, exc)
1002+
return True
1003+
1004+
1005+
def resolved_backend_for_model(model_name: str, token: str | bool | None = None) -> str:
9121006
"""Backend a fresh operation for ``model_name`` would actually select."""
9131007
raw = _raw_backend()
9141008
forced = _forced_backends.get(model_name)
9151009
key = forced or (_resolve_auto_for_model(model_name) if raw in _AUTO_ALIASES else raw)
916-
if key in _ST_ALIASES and not sentence_transformers_runtime_available():
917-
# Without a real llama binary ST is the only possible plan, and its eventual error is more useful
918-
# than a fabricated GGUF destination.
919-
if _llama_server_runtime_available():
1010+
# Without a llama binary ST is the only plan; its error beats a fabricated GGUF destination. Stat before Hub.
1011+
if key in _ST_ALIASES and _llama_server_runtime_available():
1012+
st_unusable = not sentence_transformers_runtime_available() or (
1013+
# An ST plan for it never loads, and its pending marker blocks the llama fallback's GGUF.
1014+
# Auto only: an explicit ST policy ignores the stored backend.
1015+
raw in _AUTO_ALIASES and not sentence_transformers_can_load(model_name, token)
1016+
)
1017+
if st_unusable:
9201018
key = "llama-server"
9211019
if key in _LLAMA_ALIASES:
9221020
return "llama-server"

0 commit comments

Comments
 (0)