@@ -87,6 +87,44 @@ export function createEntityStreamAdapter({ llm, agentDefaultModel, logger }) {
8787 } ;
8888}
8989
90+ /**
91+ * Issue #128: bounded backfill of rows still missing an embedding (active rows
92+ * only — needsEmbedding filters archived/forgotten). Exported for tests.
93+ *
94+ * Runs regardless of the model fingerprint: the old call-site gate returned
95+ * early when vector_meta already held the embedder's hash, permanently
96+ * orphaning rows whose embed failed at write time (embedder not ready /
97+ * provider rate limit) — one successful embed was enough to never backfill
98+ * again. markModel is idempotent when the fingerprint already matches, so
99+ * re-running costs nothing beyond the actually-missing rows.
100+ */
101+ export async function backfillMissingEmbeddings ( {
102+ store, embedder, vectorIndex, logger,
103+ maxTotal = 500 , batchSize = 10 , rateLimitMs = 200
104+ } ) {
105+ let indexed = 0 ;
106+ for ( let done = 0 ; done < maxTotal ; ) {
107+ const rows = store . needsEmbedding ( batchSize ) ;
108+ if ( ! rows . length ) break ;
109+ for ( const row of rows ) {
110+ try {
111+ const text = [ row . title , row . content ] . filter ( Boolean ) . join ( "\n" ) ;
112+ const vector = await embedder . embedSingle ( text ) ;
113+ if ( vector ?. length ) {
114+ store . setEmbedding ( row . id , vector ) ;
115+ indexed ++ ;
116+ }
117+ } catch { /* skip the bad row */ }
118+ }
119+ done += rows . length ;
120+ // Rate limit: space out batches so the provider is not hammered.
121+ if ( store . needsEmbedding ( 1 ) . length ) await new Promise ( ( r ) => setTimeout ( r , rateLimitMs ) ) ;
122+ }
123+ if ( indexed > 0 && embedder . modelHash ) vectorIndex . markModel ?. ( embedder . modelHash , embedder . dimension ) ;
124+ logger ?. info ?. ( `[dsh-mneme] auto-reindex backfilled ${ indexed } embeddings on boot` ) ;
125+ return indexed ;
126+ }
127+
90128export const apply = ( ctx , config ) => {
91129 const rawCfg = Config ( config ) ;
92130
@@ -303,10 +341,10 @@ export const apply = (ctx, config) => {
303341
304342 // Bug2: lazy auto-backfill of missing embeddings on boot. When the vector API
305343 // is configured and rows still lack an embedding (e.g. written before vector
306- // search was enabled) AND the vector_meta fingerprint is absent or stale, the
307- // index is rebuilt in the background after a short delay. Gated on
308- // cfg.autoReindexOnBoot; rate-limited in small batches so a large backlog
309- // never floods the provider. Failures degrade silently — search stays keyword.
344+ // search was enabled), the backfill runs in the background after a short
345+ // delay. Gated on cfg.autoReindexOnBoot; rate-limited in small batches so a
346+ // large backlog never floods the provider. Failures degrade silently —
347+ // search stays keyword.
310348 function scheduleAutoReindex ( ) {
311349 if ( cfg . autoReindexOnBoot === false ) return ;
312350 const attempt = ( tries ) => {
@@ -319,36 +357,13 @@ export const apply = (ctx, config) => {
319357 return ;
320358 }
321359 if ( ! store . needsEmbedding ( 1 ) . length ) return ; // nothing to backfill
322- // Model fingerprint gate: vectors already produced by the same model
323- // mean there is no drift and no rebuild needed.
324- const current = embedder . modelHash ;
325- if ( current && vectorIndex . modelHash ?. ( ) === current ) return ;
326- const BATCH = 10 ;
327- const MAX_TOTAL = 500 ; // bound boot-time work
328- ( async ( ) => {
329- let indexed = 0 ;
330- for ( let done = 0 ; done < MAX_TOTAL ; ) {
331- const rows = store . needsEmbedding ( BATCH ) ;
332- if ( ! rows . length ) break ;
333- for ( const row of rows ) {
334- try {
335- const text = [ row . title , row . content ] . filter ( Boolean ) . join ( "\n" ) ;
336- const vector = await embedder . embedSingle ( text ) ;
337- if ( vector ?. length ) {
338- store . setEmbedding ( row . id , vector ) ;
339- indexed ++ ;
340- }
341- } catch { /* skip the bad row */ }
342- }
343- done += rows . length ;
344- // Rate limit: space out batches so the provider is not hammered.
345- if ( store . needsEmbedding ( 1 ) . length ) await new Promise ( ( r ) => setTimeout ( r , 200 ) ) ;
346- }
347- if ( indexed > 0 && current ) vectorIndex . markModel ?. ( current , embedder . dimension ) ;
348- ctx . logger ?. info ?. ( `[dsh-mneme] auto-reindex backfilled ${ indexed } embeddings on boot` ) ;
349- } ) ( ) . catch ( ( error ) => {
350- ctx . logger ?. warn ?. ( `[dsh-mneme] auto-reindex failed: ${ String ( error ) } ` ) ;
351- } ) ;
360+ // Issue #128: no fingerprint gate here anymore — a matching fingerprint
361+ // used to return early and permanently orphan rows whose embed failed
362+ // at write time. See backfillMissingEmbeddings().
363+ backfillMissingEmbeddings ( { store, embedder, vectorIndex, logger : ctx . logger } )
364+ . catch ( ( error ) => {
365+ ctx . logger ?. warn ?. ( `[dsh-mneme] auto-reindex failed: ${ String ( error ) } ` ) ;
366+ } ) ;
352367 } catch ( error ) {
353368 ctx . logger ?. warn ?. ( `[dsh-mneme] auto-reindex failed: ${ String ( error ) } ` ) ;
354369 }
0 commit comments