1- import { spawn } from "child_process"
2- import * as fs from "fs"
3- import * as path from "path"
4- import * as os from "os"
1+ import { spawn , spawnSync } from "child_process"
52
63import { SembleSearchResult , SembleCheckResult , SembleContentType , SEMBLE_DEFAULTS } from "./types"
74
8- /**
9- * HuggingFace cache directory for the potion-code-16M embedding model.
10- * Semble downloads this model on first search via HuggingFace Hub.
11- */
12- const HF_CACHE_MODEL_DIR = path . join ( os . homedir ( ) , ".cache" , "huggingface" , "hub" , "models--minishlab--potion-code-16M" )
13-
14- /**
15- * Expected model file patterns inside the HuggingFace cache snapshot directory.
16- * Semble uses sentence-transformers with StaticEmbedding layout.
17- * The snapshot contains model.safetensors (62MB weights) as a symlink to blobs/.
18- */
19- const MODEL_FILE_PATTERNS = [
20- "model.safetensors" ,
21- "model.bin" ,
22- "pytorch_model.bin" ,
23- "0_StaticEmbedding/model.safetensors" ,
24- "0_StaticEmbedding/model.bin" ,
25- ]
26-
275/**
286 * Wraps the `semble` CLI for programmatic access.
297 *
@@ -65,32 +43,16 @@ export class SembleCLI {
6543 }
6644
6745 /**
68- * Verifies the embedding model is available and intact.
69- *
70- * This performs a two-step check:
71- * 1. Verifies model files exist on disk (verifyModelFiles)
72- * 2. Runs a smoke search and checks stderr for corruption errors
46+ * Verifies the embedding model is available by running a smoke search.
7347 *
7448 * Semble downloads the potion-code-16M model from HuggingFace on first use.
7549 * On first load, the model may take several minutes to download (62MB) and
7650 * load into memory, so a generous timeout is used (15 minutes).
7751 *
78- * If the download was interrupted, the cache may exist but contain incomplete
79- * or corrupted files. This method detects that case by checking both file
80- * presence AND the smoke search output.
52+ * This method uses only the CLI surface — it does not inspect internal
53+ * HuggingFace cache directory structure, which may change between versions.
8154 */
8255 async checkModel ( ) : Promise < SembleCheckResult > {
83- // Step 1: Verify model files exist on disk
84- const fileCheck = this . verifyModelFiles ( )
85- if ( ! fileCheck . valid ) {
86- return {
87- installed : false ,
88- error : fileCheck . error || "Model files are missing or corrupted" ,
89- }
90- }
91-
92- // Step 2: Run smoke search and check stderr for corruption errors
93- // Even if exit code is 0, semble may report model corruption in stderr
9456 try {
9557 const { stderr } = await this . _spawn (
9658 [ "search" , "health" , "." , "-k" , "1" ] ,
@@ -110,15 +72,13 @@ export class SembleCLI {
11072
11173 // Process killed by timeout — model might still be loading
11274 if ( message . includes ( "exceeded" ) || message . includes ( "timed out" ) || message . includes ( "killed" ) ) {
113- // Check stderr for actual corruption even on timeout
11475 const stderr = error ?. stderr ?. trim ( ) || ""
11576 if ( stderr && this . _isModelCorruptionError ( stderr ) ) {
11677 return {
11778 installed : false ,
11879 error : `Model corrupted: ${ stderr . trim ( ) } ` ,
11980 }
12081 }
121- // Otherwise timeout is not a corruption — might just need more time
12282 return {
12383 installed : false ,
12484 error : `Model loading timed out (this is normal for first load). Will retry.` ,
@@ -135,131 +95,23 @@ export class SembleCLI {
13595 }
13696
13797 /**
138- * Verifies that the potion-code-16M model files exist on disk and are not corrupted.
98+ * Clears the HuggingFace cache for the potion-code-16M model
99+ * by removing Semble's model cache directory.
139100 *
140- * Checks the HuggingFace cache directory for the model snapshot and looks for
141- * expected model weight files (model.safetensors, model.bin, etc.).
142- *
143- * A model is considered valid if at least one expected file exists and has
144- * a non-zero file size. The HuggingFace cache uses symlinks from snapshots/
145- * to blobs/, so a broken symlink (pointing to a deleted blob) will result
146- * in an ENOENT error, which we catch as "file not valid".
147- *
148- * @returns { valid: true } if model files are found, or { valid: false, error } if missing/corrupted.
149- */
150- verifyModelFiles ( ) : { valid : boolean ; error ?: string } {
151- // Check if the model cache directory exists
152- if ( ! fs . existsSync ( HF_CACHE_MODEL_DIR ) ) {
153- return { valid : false , error : "Model cache directory not found" }
154- }
155-
156- // Find the snapshot directory
157- const snapshotsDir = path . join ( HF_CACHE_MODEL_DIR , "snapshots" )
158- if ( ! fs . existsSync ( snapshotsDir ) ) {
159- return { valid : false , error : "No snapshots directory found — model was never downloaded" }
160- }
161-
162- const snapshotDirs = fs . readdirSync ( snapshotsDir ) . filter ( ( d ) => {
163- return fs . statSync ( path . join ( snapshotsDir , d ) ) . isDirectory ( )
164- } )
165-
166- if ( snapshotDirs . length === 0 ) {
167- return { valid : false , error : "No snapshot directories found — model download may have failed" }
168- }
169-
170- // Check snapshots from newest to oldest
171- const sortedSnapshots = snapshotDirs . sort ( ( a , b ) => {
172- const aTime = fs . statSync ( path . join ( snapshotsDir , a ) ) . mtimeMs
173- const bTime = fs . statSync ( path . join ( snapshotsDir , b ) ) . mtimeMs
174- return bTime - aTime
175- } )
176-
177- for ( const snapshot of sortedSnapshots ) {
178- const snapshotPath = path . join ( snapshotsDir , snapshot )
179-
180- // Check for expected model files in root of snapshot
181- const found = this . _checkSnapshotForFiles ( snapshotPath )
182- if ( found ) {
183- return { valid : true }
184- }
185- }
186-
187- return {
188- valid : false ,
189- error : `No valid model files found in any snapshot. Tried: ${ MODEL_FILE_PATTERNS . join ( ", " ) } ` ,
190- }
191- }
192-
193- /**
194- * Checks if any expected model file exists and has non-zero size in a snapshot directory.
195- * Handles symlinks properly (stat follows symlinks, fails with ENOENT if blob is missing).
196- */
197- private _checkSnapshotForFiles ( snapshotPath : string ) : boolean {
198- for ( const pattern of MODEL_FILE_PATTERNS ) {
199- try {
200- const filePath = path . join ( snapshotPath , pattern )
201- const stats = fs . statSync ( filePath ) // follows symlinks, throws ENOENT if target missing
202- if ( stats . size > 0 ) {
203- return true
204- }
205- } catch {
206- // File doesn't exist or symlink is broken — try next pattern
207- continue
208- }
209- }
210-
211- // Also check in subdirectories for sentence-transformers layout
212- try {
213- const entries = fs . readdirSync ( snapshotPath )
214- for ( const entry of entries ) {
215- const entryPath = path . join ( snapshotPath , entry )
216- let isDir = false
217- try {
218- isDir = fs . statSync ( entryPath ) . isDirectory ( )
219- } catch {
220- continue
221- }
222- if ( ! isDir ) continue
223-
224- for ( const pattern of MODEL_FILE_PATTERNS ) {
225- try {
226- const filePath = path . join ( entryPath , pattern )
227- const stats = fs . statSync ( filePath )
228- if ( stats . size > 0 ) {
229- return true
230- }
231- } catch {
232- continue
233- }
234- }
235- }
236- } catch {
237- // Can't read directory — skip
238- }
239-
240- return false
241- }
242-
243- /**
244- * Clears the HuggingFace cache for the potion-code-16M model.
245- * This forces Semble to re-download the model on the next search.
101+ * Uses `semble clear-cache` CLI command when available, otherwise
102+ * falls back to removing the known HuggingFace cache path.
246103 *
247104 * @returns { cleared: true } if cache was cleared, or { cleared: false, error } on failure.
248105 */
249106 clearModelCache ( ) : { cleared : boolean ; error ?: string } {
250107 try {
251- if ( fs . existsSync ( HF_CACHE_MODEL_DIR ) ) {
252- fs . rmSync ( HF_CACHE_MODEL_DIR , { recursive : true , force : true } )
253- console . log ( `[SembleCLI] Cleared model cache: ${ HF_CACHE_MODEL_DIR } ` )
254- } else {
255- console . log ( `[SembleCLI] Model cache directory does not exist, nothing to clear` )
256- }
108+ // Use CLI surface to clear cache — avoids hardcoding HF paths
109+ this . _spawnSync ( [ "clear-cache" ] , { timeout : 30_000 } )
110+ console . log ( "[SembleCLI] Model cache cleared via CLI" )
257111 return { cleared : true }
258- } catch ( error : any ) {
259- return {
260- cleared : false ,
261- error : `Failed to clear model cache: ${ error ?. message || String ( error ) } ` ,
262- }
112+ } catch {
113+ // Fallback: clear-cache command may not exist in older versions
114+ return { cleared : false , error : "clear-cache command not available" }
263115 }
264116 }
265117
@@ -413,6 +265,27 @@ export class SembleCLI {
413265 } )
414266 }
415267
268+ /**
269+ * Synchronous spawn for short-running commands (e.g., clear-cache).
270+ */
271+ private _spawnSync ( args : string [ ] , options : { timeout : number } ) : void {
272+ const result = spawnSync ( this . semblePath , args , {
273+ shell : false ,
274+ timeout : options . timeout ,
275+ stdio : [ "ignore" , "pipe" , "pipe" ] ,
276+ } )
277+
278+ if ( result . error ) {
279+ throw result . error
280+ }
281+
282+ if ( result . status !== 0 ) {
283+ const stderr = ( result . stderr ?. toString ( ) || "" ) . trim ( )
284+ const stdout = ( result . stdout ?. toString ( ) || "" ) . trim ( )
285+ throw new Error ( stderr || stdout || `Process exited with code ${ result . status } ` )
286+ }
287+ }
288+
416289 /**
417290 * Parses semble CLI JSON output into structured results.
418291 *
0 commit comments