@@ -98,6 +98,16 @@ const CHUNKDB_WRITE_SLICE_BYTES = 512 * 1024
9898const CHUNKDB_WRITE_SLICE_PAUSE_MS = 15
9999const CHUNKDB_BUSY_RETRY_MS = 35
100100const BACKLOG_SAMPLE_ROWS = 2_048
101+ // Once a historical backfill drains, a giant freelist can remain inside the
102+ // SQLite file even though the logical payload has already shrunk dramatically.
103+ // Keep normal maintenance conservative, but enter a dedicated low-priority
104+ // reclaim drain until the freelist falls below this ratio. Vacuum work remains
105+ // split into small statements with yields so foreground writers keep priority.
106+ const CHUNKDB_RECLAIM_DRAIN_FREE_RATIO = 0.05
107+ const CHUNKDB_RECLAIM_DRAIN_MIN_FREE_PAGES = 8_192 // 64 MiB at the tuned 8 KiB page size
108+ const CHUNKDB_RECLAIM_DRAIN_MAX_MS = 15_000
109+ const CHUNKDB_RECLAIM_DRAIN_PAUSE_MS = 10
110+ const CHUNKDB_RECLAIM_DRAIN_SLEEP_MS = 250
101111
102112// A Database layer can be materialized more than once inside one server process
103113// (for example by independently-built service graphs). Starting one infinite
@@ -171,14 +181,91 @@ interface SealerOptions {
171181 * statement. On DBs without INCREMENTAL auto_vacuum (existing files) this is a
172182 * cheap no-op. Must run OUTSIDE a transaction.
173183 */
174- function reclaimSpace ( db : DatabaseShape ) : Effect . Effect < void > {
184+ type ReclaimMode = "maintenance" | "drain"
185+
186+ interface ReclaimStats {
187+ readonly attempted : boolean
188+ readonly reclaimedPages : number
189+ readonly freePages : number
190+ readonly pageCount : number
191+ readonly needsMore : boolean
192+ }
193+
194+ export function shouldDrainFreelist ( freePages : number , pageCount : number ) : boolean {
195+ if ( freePages < CHUNKDB_RECLAIM_DRAIN_MIN_FREE_PAGES || pageCount <= 0 ) return false
196+ return freePages / pageCount > CHUNKDB_RECLAIM_DRAIN_FREE_RATIO
197+ }
198+
199+ function reclaimSpace ( db : DatabaseShape , mode : ReclaimMode = "maintenance" ) : Effect . Effect < ReclaimStats > {
175200 return Effect . gen ( function * ( ) {
176201 const rows = yield * db . all < { auto_vacuum : number } > ( `PRAGMA auto_vacuum` ) . pipe ( Effect . orDie )
177- if ( ( rows [ 0 ] ?. auto_vacuum ?? 0 ) !== 2 ) return
178- for ( let i = 0 ; i < CHUNKDB_VACUUM_MAX_ITERATIONS ; i ++ ) {
179- const free = yield * db . all < { freelist_count : number } > ( `PRAGMA freelist_count` ) . pipe ( Effect . orDie )
180- if ( ( free [ 0 ] ?. freelist_count ?? 0 ) === 0 ) break
181- yield * db . run ( `PRAGMA incremental_vacuum(${ CHUNKDB_VACUUM_PAGES_PER_PASS } )` ) . pipe ( Effect . orDie )
202+ if ( ( rows [ 0 ] ?. auto_vacuum ?? 0 ) !== 2 ) {
203+ return { attempted : false , reclaimedPages : 0 , freePages : 0 , pageCount : 0 , needsMore : false }
204+ }
205+
206+ const initialFreeRows = yield * db . all < { freelist_count : number } > ( `PRAGMA freelist_count` ) . pipe ( Effect . orDie )
207+ const initialPageRows = yield * db . all < { page_count : number } > ( `PRAGMA page_count` ) . pipe ( Effect . orDie )
208+ const initialFree = initialFreeRows [ 0 ] ?. freelist_count ?? 0
209+ const initialPageCount = initialPageRows [ 0 ] ?. page_count ?? 0
210+ if ( initialFree === 0 ) {
211+ return { attempted : true , reclaimedPages : 0 , freePages : 0 , pageCount : initialPageCount , needsMore : false }
212+ }
213+
214+ const started = Date . now ( )
215+ const maxIterations = mode === "maintenance" ? CHUNKDB_VACUUM_MAX_ITERATIONS : Number . POSITIVE_INFINITY
216+ let iterations = 0
217+ let currentFree = initialFree
218+ let currentPageCount = initialPageCount
219+
220+ while ( iterations < maxIterations ) {
221+ if ( mode === "drain" ) {
222+ if ( ! shouldDrainFreelist ( currentFree , currentPageCount ) ) break
223+ if ( Date . now ( ) - started >= CHUNKDB_RECLAIM_DRAIN_MAX_MS ) break
224+ } else if ( currentFree === 0 ) {
225+ break
226+ }
227+
228+ const vacuum = yield * db . run ( `PRAGMA incremental_vacuum(${ CHUNKDB_VACUUM_PAGES_PER_PASS } )` ) . pipe (
229+ Effect . map ( ( ) => true ) ,
230+ Effect . catch ( ( error ) => {
231+ if ( isSqliteBusy ( error ) ) return Effect . succeed ( false )
232+ return Effect . die ( error )
233+ } ) ,
234+ )
235+ if ( ! vacuum ) {
236+ // Foreground owns the single SQLite writer slot. Background reclaim does
237+ // not queue behind it; yield immediately and let the next drain slice try.
238+ break
239+ }
240+
241+ iterations += 1
242+ const freeRows = yield * db . all < { freelist_count : number } > ( `PRAGMA freelist_count` ) . pipe ( Effect . orDie )
243+ const pageRows = yield * db . all < { page_count : number } > ( `PRAGMA page_count` ) . pipe ( Effect . orDie )
244+ const nextFree = freeRows [ 0 ] ?. freelist_count ?? 0
245+ const nextPageCount = pageRows [ 0 ] ?. page_count ?? currentPageCount
246+
247+ // Defensive escape hatch: a driver/SQLite combination that reports no
248+ // reclaim progress must never turn the background loop into a CPU spin.
249+ if ( nextFree >= currentFree && nextPageCount >= currentPageCount ) {
250+ currentFree = nextFree
251+ currentPageCount = nextPageCount
252+ break
253+ }
254+
255+ currentFree = nextFree
256+ currentPageCount = nextPageCount
257+ yield * Effect . yieldNow
258+ if ( mode === "drain" ) yield * Effect . sleep ( Duration . millis ( CHUNKDB_RECLAIM_DRAIN_PAUSE_MS ) )
259+ }
260+
261+ const needsMore = mode === "drain" && shouldDrainFreelist ( currentFree , currentPageCount )
262+
263+ return {
264+ attempted : true ,
265+ reclaimedPages : Math . max ( 0 , initialFree - currentFree ) ,
266+ freePages : currentFree ,
267+ pageCount : currentPageCount ,
268+ needsMore,
182269 }
183270 } )
184271}
@@ -555,6 +642,13 @@ export function runPassV2(
555642 const frameByID = new Map ( compressed )
556643
557644 const plans : Plan [ ] = [ ]
645+ // Delta matching is useful for modest record-shaped values, but it is the
646+ // wrong representation for jumbo rows. Besides adding read amplification,
647+ // building a correction requires indexing/comparing both raw values. Keep
648+ // jumbo payloads on the independently-decodable v4 segmented frame path.
649+ // This also makes a corrupted/hostile giant event incapable of monopolizing
650+ // a background sealing pass when delta mode is explicitly enabled.
651+ const deltaMaxRawBytes = 4 * 1024 * 1024
558652 for ( const plan of pending ) {
559653 if ( plan . kind === "skip" ) {
560654 plans . push ( plan )
@@ -573,9 +667,13 @@ export function runPassV2(
573667
574668 const frame = frameByID . get ( candidate . id ) ?? candidate . data
575669 let finalFrame : string | Uint8Array = frame
576- if ( Flag . OPENCODE_SEAL_DELTA && typeof frame !== "string" ) {
670+ if (
671+ Flag . OPENCODE_SEAL_DELTA &&
672+ typeof frame !== "string" &&
673+ rawBytes . byteLength <= deltaMaxRawBytes
674+ ) {
577675 const base = lastBaseByAggregate . get ( candidate . aggregate_id )
578- if ( base !== undefined ) {
676+ if ( base !== undefined && base . raw . byteLength <= deltaMaxRawBytes ) {
579677 const { codec, level } = chooseCodec ( rawBytes . byteLength )
580678 const delta = compressDeltaRef ( rawBytes , base . raw , base . valueId , codec , level )
581679 if ( delta . byteLength < frame . byteLength * 0.7 ) finalFrame = delta
@@ -868,6 +966,7 @@ export function runSealerLoop(filename: string): Effect.Effect<void> {
868966 hotTailEvents : CHUNKDB_HOT_TAIL_EVENTS ,
869967 } )
870968 let previousHitCap : boolean = false
969+ let reclaimDraining = false
871970 let backoffMs = BACKOFF_BASE_MS
872971 for ( ; ; ) {
873972 const draining : boolean = previousHitCap && backfillAllowed
@@ -914,7 +1013,36 @@ export function runSealerLoop(filename: string): Effect.Effect<void> {
9141013 )
9151014 yield * Effect . logDebug ( "ChunkDB sealer idle" , { filename, ...backlog } )
9161015 }
917- const wait = previousHitCap && backfillAllowed ? DRAIN_SLEEP_MS : MAINTENANCE_INTERVAL_MS
1016+
1017+ // When sealing no longer has a capped backlog, immediately inspect
1018+ // the physical freelist. A multi-GB historical backfill can leave the
1019+ // logical DB tiny while SQLite still owns gigabytes of free pages.
1020+ // Drain those pages in low-priority 15s slices until the free-space
1021+ // ratio is sane instead of waiting ten minutes between ~160MB
1022+ // maintenance reclaims. This also repairs a DB that finished its
1023+ // backfill in a previous process before this policy existed.
1024+ if ( ! previousHitCap ) {
1025+ const reclaim = yield * reclaimSpace ( db , "drain" )
1026+ reclaimDraining = reclaim . needsMore
1027+ if ( reclaim . reclaimedPages > 0 ) {
1028+ yield * Effect . logInfo ( "ChunkDB space reclaim progress" , {
1029+ filename,
1030+ reclaimedPages : reclaim . reclaimedPages ,
1031+ freePages : reclaim . freePages ,
1032+ pageCount : reclaim . pageCount ,
1033+ freeRatio : reclaim . pageCount > 0 ? reclaim . freePages / reclaim . pageCount : 0 ,
1034+ draining : reclaim . needsMore ,
1035+ } )
1036+ }
1037+ } else {
1038+ reclaimDraining = false
1039+ }
1040+
1041+ const wait = previousHitCap && backfillAllowed
1042+ ? DRAIN_SLEEP_MS
1043+ : reclaimDraining
1044+ ? CHUNKDB_RECLAIM_DRAIN_SLEEP_MS
1045+ : MAINTENANCE_INTERVAL_MS
9181046 yield * Effect . sleep ( Duration . millis ( wait ) )
9191047 }
9201048 } ) ,
0 commit comments