Skip to content

Commit 8347944

Browse files
cdeustclaude
andcommitted
feat(embedding): zero-download algorithmic fallback on the #173 seams (#169)
Phase B — reimplements #169 on the merged provider/lifecycle/factory seams. 1. Second provider: AlgorithmicEmbeddingProvider (embedding_fallback_provider.py) implements EmbeddingProvider via shared.algorithmic_embedding (TF + Random Indexing + co-occurrence bridging). The factory selects per ModelState — embedding_factory.use_fallback maps every non-LOADED state (PACKAGE_ABSENT, MODEL_FILES_ABSENT, LOAD_RAISED, UNINITIALIZED) to the fallback and LOADED to neural; the mapping is total (no state maps to nothing). The lifecycle no longer blocks or crashes: MODEL_FILES_ABSENT and LOAD_RAISED serve fallback this session and fetch in the background (embedding_downloads.py), upgrading next session. 2. Root-cause fix: SqliteMemoryStore stamps embedding_model UNCONDITIONALLY on write (outside the _has_vec branch) — 'fallback' even with sqlite-vec absent, never ''. 3. Loud: one WARNING when the fallback engages + get_telemetry embedding_mode. 4. Code-aware FTS on all recall paths: camelCase/snake_case sub-tokens at index time (augment_content) and query time (recall WRRF _signal_fts, entity MATCH, and recall_helpers.compute_vector_fts). memories_fts converted to self-content with a one-shot rebuild+reindex migration. 5. Scheduled upgrade: consolidation/embedding_upgrade.run_embedding_upgrade_cycle re-embeds fallback-stamped memories with neural once available and restamps them, wired into consolidate's compress phase; count in the response. 6/8. Mixed store never cross-ranks (query-space vec filter); genuine sentence-transformers absence + empty HF_HOME covered by a subprocess test. Co-Authored-By: Claude Fable 5 <[email protected]>
1 parent fc33d88 commit 8347944

18 files changed

Lines changed: 1510 additions & 149 deletions

‎mcp_server/handlers/consolidate.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,9 @@
1515
from mcp_server.handlers.consolidation.cls import run_cls_cycle
1616
from mcp_server.handlers.consolidation.compression import run_compression_cycle
1717
from mcp_server.handlers.consolidation.decay import run_decay_cycle
18+
from mcp_server.handlers.consolidation.embedding_upgrade import (
19+
run_embedding_upgrade_cycle,
20+
)
1821
from mcp_server.handlers.consolidation.entity_merge import run_entity_merge_cycle
1922
from mcp_server.handlers.consolidation.forgetting import run_forgetting_cycle
2023
from mcp_server.handlers.consolidation.homeostatic import run_homeostatic_cycle
@@ -339,6 +342,14 @@ def _run_cycles(
339342
if args.get("cls", True):
340343
stats["cls"] = _timed(run_cls_cycle, store, settings, embeddings)
341344

345+
# Transparent #169 upgrade: re-embed memories written by the download-free
346+
# fallback once the neural model is available, restamping them 'neural'.
347+
# Runs with the compress phase (both re-embed on the current encoder).
348+
if args.get("compress", True):
349+
stats["embedding_upgrade"] = _timed(
350+
run_embedding_upgrade_cycle, store, embeddings
351+
)
352+
342353
if args.get("memify", True):
343354
stats["memify"] = _timed(run_memify_cycle, store, None)
344355

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
"""Embedding-upgrade cycle: re-embed fallback memories once the model arrives.
2+
3+
The scheduled other half of the #169 zero-download fallback. When a session ran
4+
on the download-free algorithmic provider (``embedding_model = 'fallback'``) and
5+
a later session finds the neural model present, this maintenance cycle re-embeds
6+
those memories with the neural encoder and restamps them 'neural' — so the
7+
store transparently upgrades to full-fidelity vectors over time instead of
8+
leaving two incompatible spaces side by side forever.
9+
10+
Bounded per run (``_MAX_UPGRADE_PER_CYCLE``) so a large backlog is drained
11+
across several consolidations rather than in one blocking pass, mirroring the
12+
other streaming cycles. No-op when the current encoder is still fallback (no
13+
neural model to upgrade to) or when the store has no fallback worklist.
14+
"""
15+
16+
from __future__ import annotations
17+
18+
import logging
19+
from typing import Any
20+
21+
from mcp_server.infrastructure.embedding_engine import EmbeddingEngine
22+
from mcp_server.infrastructure.memory_store import MemoryStore
23+
24+
logger = logging.getLogger(__name__)
25+
26+
# source: matches sleep_compute.run_sleep_compute_streamed's max_reembed=100
27+
# bound (mcp_server/core/sleep_compute.py) — the same per-cycle re-embed budget
28+
# already used for stale/compressed embeddings, reused here for parity.
29+
_MAX_UPGRADE_PER_CYCLE = 100
30+
31+
32+
def run_embedding_upgrade_cycle(
33+
store: MemoryStore, embeddings: EmbeddingEngine
34+
) -> dict[str, Any]:
35+
"""Re-embed up to ``_MAX_UPGRADE_PER_CYCLE`` fallback memories with neural.
36+
37+
precondition: ``embeddings`` is the process encoder; ``store`` may or may
38+
not expose the #169 fallback worklist.
39+
postcondition: returns ``{"upgraded": n}`` (and a ``reason`` when it skips).
40+
Only runs when the encoder currently resolves to the neural model; each
41+
upgraded memory's vector is replaced and its ``embedding_model`` restamped
42+
'neural' via ``store.reembed_memory``. Non-fatal: any failure is logged and
43+
reported as zero upgrades.
44+
"""
45+
if not hasattr(store, "select_fallback_embeddings") or not hasattr(
46+
store, "reembed_memory"
47+
):
48+
return {"upgraded": 0, "reason": "store has no fallback worklist"}
49+
if getattr(embeddings, "mode", "neural") != "neural":
50+
return {"upgraded": 0, "reason": "no neural model available yet"}
51+
try:
52+
candidates = store.select_fallback_embeddings(limit=_MAX_UPGRADE_PER_CYCLE)
53+
except Exception:
54+
logger.debug("Embedding-upgrade cycle: worklist query failed (non-fatal)")
55+
return {"upgraded": 0}
56+
57+
upgraded = 0
58+
for item in candidates:
59+
content = item.get("content")
60+
if not content:
61+
continue
62+
try:
63+
emb = embeddings.encode(content)
64+
if emb:
65+
store.reembed_memory(item["memory_id"], emb)
66+
upgraded += 1
67+
except Exception:
68+
logger.debug(
69+
"Embedding-upgrade failed for memory %s (non-fatal)",
70+
item.get("memory_id"),
71+
)
72+
return {"upgraded": upgraded}

‎mcp_server/handlers/get_telemetry.py‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616

1717
from mcp_server.core import telemetry
1818
from mcp_server.handlers._tool_meta import READ_ONLY
19+
from mcp_server.infrastructure.embedding_engine import current_embedding_mode
1920

2021
schema = {
2122
"title": "Get Telemetry (read/write counters)",
@@ -59,6 +60,16 @@
5960
"environment when the process started."
6061
),
6162
},
63+
"embedding_mode": {
64+
"type": "string",
65+
"description": (
66+
"Embedding provenance for this process (issue #169): "
67+
"'neural' = sentence-transformers, 'fallback' = "
68+
"download-free algorithmic embeddings (lower fidelity, "
69+
"engaged when the model is absent), 'unknown' = no encode "
70+
"has run yet. Fallback and neural vectors never cross-rank."
71+
),
72+
},
6273
},
6374
},
6475
"description": (
@@ -79,6 +90,10 @@ async def handler(args: dict[str, Any] | None = None) -> dict[str, Any]:
7990
"""Return current telemetry summary.
8091
8192
precondition: none (read-only over in-memory dict).
82-
postcondition: returns ``telemetry.summary()`` verbatim.
93+
postcondition: returns ``telemetry.summary()`` augmented with
94+
``embedding_mode`` (issue #169) so a caller can tell whether semantic recall
95+
is running on neural or download-free fallback embeddings.
8396
"""
84-
return telemetry.summary()
97+
summary = telemetry.summary()
98+
summary["embedding_mode"] = current_embedding_mode()
99+
return summary

‎mcp_server/handlers/recall_helpers.py‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010

1111
from mcp_server.core import thermodynamics
1212
from mcp_server.core.enrichment import build_expanded_query
13+
from mcp_server.shared.code_tokenize import expand_fts_query
1314
from mcp_server.core.prospective import check_trigger
1415
from mcp_server.core.query_intent import QueryIntent
1516
from mcp_server.core.retrieval_signals import (
@@ -44,6 +45,17 @@ def compute_vector_fts(
4445
fts.extend(
4546
(m, s) for m, s in store.search_fts(query, limit=pool // 2) if m not in ids
4647
)
48+
# Code-aware discovery (issue #169): a camelCase/snake_case query term is
49+
# expanded into its sub-tokens so a lowercase query ("payment") recalls a
50+
# symbol memory ("normalizePaymentAmount") on the SQLite FTS path — and vice
51+
# versa. Merge-in only (never removes a hit); the raw/expanded searches
52+
# above stay primary, so this is additive on both backends.
53+
code_q = expand_fts_query(query)
54+
if code_q and code_q not in (query, expanded):
55+
ids = {m for m, _ in fts}
56+
fts.extend(
57+
(m, s) for m, s in store.search_fts(code_q, limit=pool // 2) if m not in ids
58+
)
4759
return vec, fts, q_emb
4860

4961

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
"""Best-effort background fetches for the embedding subsystem (issue #169).
2+
3+
Non-blocking helpers the model lifecycle uses when it engages the download-free
4+
fallback: kick off the package install (PACKAGE_ABSENT) or the model-weight
5+
download (MODEL_FILES_ABSENT) in a detached subprocess so the current session
6+
serves fallback embeddings immediately and the neural model becomes available
7+
for the NEXT session — the "no first-run cliff" behaviour of #169.
8+
9+
Both helpers honour ``CORTEX_EMBEDDING_ZERO_DOWNLOAD``: when set, no network
10+
fetch is started at all (sandboxed / offline / test use). All failures are
11+
swallowed — a background fetch that cannot start must never affect the caller.
12+
13+
Pure I/O (subprocess). Infrastructure layer.
14+
"""
15+
16+
from __future__ import annotations
17+
18+
import logging
19+
import os
20+
21+
logger = logging.getLogger(__name__)
22+
23+
24+
def zero_download_requested() -> bool:
25+
"""Whether the operator opted out of any embedding network fetch (#169).
26+
27+
source: opt-in env contract — ``CORTEX_EMBEDDING_ZERO_DOWNLOAD`` in
28+
{``1``, ``true``} suppresses both the background package install and the
29+
background model download, forcing pure download-free operation.
30+
"""
31+
return os.environ.get("CORTEX_EMBEDDING_ZERO_DOWNLOAD", "").lower() in ("1", "true")
32+
33+
34+
def _spawn_detached(cmd: list[str], what: str) -> None:
35+
"""Start ``cmd`` as a detached, output-suppressed subprocess; swallow errors."""
36+
import subprocess
37+
38+
try:
39+
subprocess.Popen(
40+
cmd,
41+
stdout=subprocess.DEVNULL,
42+
stderr=subprocess.DEVNULL,
43+
start_new_session=True,
44+
)
45+
logger.info("Background %s started", what)
46+
except Exception as exc:
47+
logger.debug("Background %s failed to start: %s", what, exc)
48+
49+
50+
def trigger_background_install() -> None:
51+
"""Install sentence-transformers in the background (PACKAGE_ABSENT).
52+
53+
Next session will have the package and a real model. No-op under
54+
``CORTEX_EMBEDDING_ZERO_DOWNLOAD``.
55+
"""
56+
if zero_download_requested():
57+
return
58+
import sys
59+
60+
target = os.environ.get("CLAUDE_PLUGIN_DATA", "")
61+
if target:
62+
target = os.path.join(target, "deps")
63+
cmd = [
64+
sys.executable,
65+
"-m",
66+
"pip",
67+
"install",
68+
"-q",
69+
"sentence-transformers>=2.2.0,<4.0.0",
70+
]
71+
if target:
72+
cmd.extend(["--target", target])
73+
_spawn_detached(cmd, "install of sentence-transformers")
74+
75+
76+
def trigger_background_model_download(
77+
model_name: str, revision: str | None, cache_folder: str | None
78+
) -> None:
79+
"""Download the embedding model weights in the background (MODEL_FILES_ABSENT).
80+
81+
Populates the local HF cache so the NEXT session loads the neural model.
82+
No-op under ``CORTEX_EMBEDDING_ZERO_DOWNLOAD``. The child imports
83+
sentence-transformers itself; a failure (offline) is silently dropped —
84+
this session already serves fallback.
85+
"""
86+
if zero_download_requested():
87+
return
88+
import sys
89+
90+
# The child constructs the model once to force the download into the cache.
91+
prog = (
92+
"import sys\n"
93+
"from sentence_transformers import SentenceTransformer\n"
94+
"name, rev, cf = sys.argv[1], sys.argv[2] or None, sys.argv[3] or None\n"
95+
"SentenceTransformer(name, revision=rev, cache_folder=cf)\n"
96+
)
97+
cmd = [sys.executable, "-c", prog, model_name, revision or "", cache_folder or ""]
98+
_spawn_detached(cmd, f"download of embedding model {model_name}")

0 commit comments

Comments
 (0)