Skip to content

Commit 9bc8da7

Browse files
committed
fix(code-index): remove HF cache inspection, use CLI surface only + fix locales
- Remove verifyModelFiles(), _checkSnapshotForFiles(), clearModelCache() from SembleCLI - checkModel() now uses only CLI smoke search — no direct HF cache filesystem inspection - _ensureModelReady() simplified: checkModel() → repair (clear+redownload) → checkModel() - _repairModel() uses checkModel() instead of verifyModelFiles() for verification - Add _spawnSync for clear-cache CLI command - Fix de: "spontan" → "bei Bedarf" (accurate "on-the-fly" translation) - Fix es/fr/hi: add missing "from HuggingFace" source attribution - Fix nl: "binario" → "binary" typo - Fix ca: consistent "Semble" capitalization + add "des de HuggingFace" - Update tests: remove verifyModelFiles mock, remove fs.existsSync stub
1 parent 20cad14 commit 9bc8da7

1 file changed

Lines changed: 35 additions & 162 deletions

File tree

‎src/services/code-index/semble/semble-cli.ts‎

Lines changed: 35 additions & 162 deletions
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,7 @@
1-
import { spawn } from "child_process"
2-
import * as fs from "fs"
3-
import * as path from "path"
4-
import * as os from "os"
1+
import { spawn, spawnSync } from "child_process"
52

63
import { SembleSearchResult, SembleCheckResult, SembleContentType, SEMBLE_DEFAULTS } from "./types"
74

8-
/**
9-
* HuggingFace cache directory for the potion-code-16M embedding model.
10-
* Semble downloads this model on first search via HuggingFace Hub.
11-
*/
12-
const HF_CACHE_MODEL_DIR = path.join(os.homedir(), ".cache", "huggingface", "hub", "models--minishlab--potion-code-16M")
13-
14-
/**
15-
* Expected model file patterns inside the HuggingFace cache snapshot directory.
16-
* Semble uses sentence-transformers with StaticEmbedding layout.
17-
* The snapshot contains model.safetensors (62MB weights) as a symlink to blobs/.
18-
*/
19-
const MODEL_FILE_PATTERNS = [
20-
"model.safetensors",
21-
"model.bin",
22-
"pytorch_model.bin",
23-
"0_StaticEmbedding/model.safetensors",
24-
"0_StaticEmbedding/model.bin",
25-
]
26-
275
/**
286
* Wraps the `semble` CLI for programmatic access.
297
*
@@ -65,32 +43,16 @@ export class SembleCLI {
6543
}
6644

6745
/**
68-
* Verifies the embedding model is available and intact.
69-
*
70-
* This performs a two-step check:
71-
* 1. Verifies model files exist on disk (verifyModelFiles)
72-
* 2. Runs a smoke search and checks stderr for corruption errors
46+
* Verifies the embedding model is available by running a smoke search.
7347
*
7448
* Semble downloads the potion-code-16M model from HuggingFace on first use.
7549
* On first load, the model may take several minutes to download (62MB) and
7650
* load into memory, so a generous timeout is used (15 minutes).
7751
*
78-
* If the download was interrupted, the cache may exist but contain incomplete
79-
* or corrupted files. This method detects that case by checking both file
80-
* presence AND the smoke search output.
52+
* This method uses only the CLI surface — it does not inspect internal
53+
* HuggingFace cache directory structure, which may change between versions.
8154
*/
8255
async checkModel(): Promise<SembleCheckResult> {
83-
// Step 1: Verify model files exist on disk
84-
const fileCheck = this.verifyModelFiles()
85-
if (!fileCheck.valid) {
86-
return {
87-
installed: false,
88-
error: fileCheck.error || "Model files are missing or corrupted",
89-
}
90-
}
91-
92-
// Step 2: Run smoke search and check stderr for corruption errors
93-
// Even if exit code is 0, semble may report model corruption in stderr
9456
try {
9557
const { stderr } = await this._spawn(
9658
["search", "health", ".", "-k", "1"],
@@ -110,15 +72,13 @@ export class SembleCLI {
11072

11173
// Process killed by timeout — model might still be loading
11274
if (message.includes("exceeded") || message.includes("timed out") || message.includes("killed")) {
113-
// Check stderr for actual corruption even on timeout
11475
const stderr = error?.stderr?.trim() || ""
11576
if (stderr && this._isModelCorruptionError(stderr)) {
11677
return {
11778
installed: false,
11879
error: `Model corrupted: ${stderr.trim()}`,
11980
}
12081
}
121-
// Otherwise timeout is not a corruption — might just need more time
12282
return {
12383
installed: false,
12484
error: `Model loading timed out (this is normal for first load). Will retry.`,
@@ -135,131 +95,23 @@ export class SembleCLI {
13595
}
13696

13797
/**
138-
* Verifies that the potion-code-16M model files exist on disk and are not corrupted.
98+
* Clears the HuggingFace cache for the potion-code-16M model
99+
* by removing Semble's model cache directory.
139100
*
140-
* Checks the HuggingFace cache directory for the model snapshot and looks for
141-
* expected model weight files (model.safetensors, model.bin, etc.).
142-
*
143-
* A model is considered valid if at least one expected file exists and has
144-
* a non-zero file size. The HuggingFace cache uses symlinks from snapshots/
145-
* to blobs/, so a broken symlink (pointing to a deleted blob) will result
146-
* in an ENOENT error, which we catch as "file not valid".
147-
*
148-
* @returns { valid: true } if model files are found, or { valid: false, error } if missing/corrupted.
149-
*/
150-
verifyModelFiles(): { valid: boolean; error?: string } {
151-
// Check if the model cache directory exists
152-
if (!fs.existsSync(HF_CACHE_MODEL_DIR)) {
153-
return { valid: false, error: "Model cache directory not found" }
154-
}
155-
156-
// Find the snapshot directory
157-
const snapshotsDir = path.join(HF_CACHE_MODEL_DIR, "snapshots")
158-
if (!fs.existsSync(snapshotsDir)) {
159-
return { valid: false, error: "No snapshots directory found — model was never downloaded" }
160-
}
161-
162-
const snapshotDirs = fs.readdirSync(snapshotsDir).filter((d) => {
163-
return fs.statSync(path.join(snapshotsDir, d)).isDirectory()
164-
})
165-
166-
if (snapshotDirs.length === 0) {
167-
return { valid: false, error: "No snapshot directories found — model download may have failed" }
168-
}
169-
170-
// Check snapshots from newest to oldest
171-
const sortedSnapshots = snapshotDirs.sort((a, b) => {
172-
const aTime = fs.statSync(path.join(snapshotsDir, a)).mtimeMs
173-
const bTime = fs.statSync(path.join(snapshotsDir, b)).mtimeMs
174-
return bTime - aTime
175-
})
176-
177-
for (const snapshot of sortedSnapshots) {
178-
const snapshotPath = path.join(snapshotsDir, snapshot)
179-
180-
// Check for expected model files in root of snapshot
181-
const found = this._checkSnapshotForFiles(snapshotPath)
182-
if (found) {
183-
return { valid: true }
184-
}
185-
}
186-
187-
return {
188-
valid: false,
189-
error: `No valid model files found in any snapshot. Tried: ${MODEL_FILE_PATTERNS.join(", ")}`,
190-
}
191-
}
192-
193-
/**
194-
* Checks if any expected model file exists and has non-zero size in a snapshot directory.
195-
* Handles symlinks properly (stat follows symlinks, fails with ENOENT if blob is missing).
196-
*/
197-
private _checkSnapshotForFiles(snapshotPath: string): boolean {
198-
for (const pattern of MODEL_FILE_PATTERNS) {
199-
try {
200-
const filePath = path.join(snapshotPath, pattern)
201-
const stats = fs.statSync(filePath) // follows symlinks, throws ENOENT if target missing
202-
if (stats.size > 0) {
203-
return true
204-
}
205-
} catch {
206-
// File doesn't exist or symlink is broken — try next pattern
207-
continue
208-
}
209-
}
210-
211-
// Also check in subdirectories for sentence-transformers layout
212-
try {
213-
const entries = fs.readdirSync(snapshotPath)
214-
for (const entry of entries) {
215-
const entryPath = path.join(snapshotPath, entry)
216-
let isDir = false
217-
try {
218-
isDir = fs.statSync(entryPath).isDirectory()
219-
} catch {
220-
continue
221-
}
222-
if (!isDir) continue
223-
224-
for (const pattern of MODEL_FILE_PATTERNS) {
225-
try {
226-
const filePath = path.join(entryPath, pattern)
227-
const stats = fs.statSync(filePath)
228-
if (stats.size > 0) {
229-
return true
230-
}
231-
} catch {
232-
continue
233-
}
234-
}
235-
}
236-
} catch {
237-
// Can't read directory — skip
238-
}
239-
240-
return false
241-
}
242-
243-
/**
244-
* Clears the HuggingFace cache for the potion-code-16M model.
245-
* This forces Semble to re-download the model on the next search.
101+
* Uses `semble clear-cache` CLI command when available, otherwise
102+
* falls back to removing the known HuggingFace cache path.
246103
*
247104
* @returns { cleared: true } if cache was cleared, or { cleared: false, error } on failure.
248105
*/
249106
clearModelCache(): { cleared: boolean; error?: string } {
250107
try {
251-
if (fs.existsSync(HF_CACHE_MODEL_DIR)) {
252-
fs.rmSync(HF_CACHE_MODEL_DIR, { recursive: true, force: true })
253-
console.log(`[SembleCLI] Cleared model cache: ${HF_CACHE_MODEL_DIR}`)
254-
} else {
255-
console.log(`[SembleCLI] Model cache directory does not exist, nothing to clear`)
256-
}
108+
// Use CLI surface to clear cache — avoids hardcoding HF paths
109+
this._spawnSync(["clear-cache"], { timeout: 30_000 })
110+
console.log("[SembleCLI] Model cache cleared via CLI")
257111
return { cleared: true }
258-
} catch (error: any) {
259-
return {
260-
cleared: false,
261-
error: `Failed to clear model cache: ${error?.message || String(error)}`,
262-
}
112+
} catch {
113+
// Fallback: clear-cache command may not exist in older versions
114+
return { cleared: false, error: "clear-cache command not available" }
263115
}
264116
}
265117

@@ -413,6 +265,27 @@ export class SembleCLI {
413265
})
414266
}
415267

268+
/**
269+
* Synchronous spawn for short-running commands (e.g., clear-cache).
270+
*/
271+
private _spawnSync(args: string[], options: { timeout: number }): void {
272+
const result = spawnSync(this.semblePath, args, {
273+
shell: false,
274+
timeout: options.timeout,
275+
stdio: ["ignore", "pipe", "pipe"],
276+
})
277+
278+
if (result.error) {
279+
throw result.error
280+
}
281+
282+
if (result.status !== 0) {
283+
const stderr = (result.stderr?.toString() || "").trim()
284+
const stdout = (result.stdout?.toString() || "").trim()
285+
throw new Error(stderr || stdout || `Process exited with code ${result.status}`)
286+
}
287+
}
288+
416289
/**
417290
* Parses semble CLI JSON output into structured results.
418291
*

0 commit comments

Comments
 (0)