Skip to content

Commit 7cd1b42

Browse files
committed
fix(core): harden ChunkDB delta and space reclaim
1 parent 41fc49a commit 7cd1b42

4 files changed

Lines changed: 169 additions & 11 deletions

File tree

‎packages/core/src/database/chunk-sealer.ts‎

Lines changed: 137 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,16 @@ const CHUNKDB_WRITE_SLICE_BYTES = 512 * 1024
9898
const CHUNKDB_WRITE_SLICE_PAUSE_MS = 15
9999
const CHUNKDB_BUSY_RETRY_MS = 35
100100
const BACKLOG_SAMPLE_ROWS = 2_048
101+
// Once a historical backfill drains, a giant freelist can remain inside the
102+
// SQLite file even though the logical payload has already shrunk dramatically.
103+
// Keep normal maintenance conservative, but enter a dedicated low-priority
104+
// reclaim drain until the freelist falls below this ratio. Vacuum work remains
105+
// split into small statements with yields so foreground writers keep priority.
106+
const CHUNKDB_RECLAIM_DRAIN_FREE_RATIO = 0.05
107+
const CHUNKDB_RECLAIM_DRAIN_MIN_FREE_PAGES = 8_192 // 64 MiB at the tuned 8 KiB page size
108+
const CHUNKDB_RECLAIM_DRAIN_MAX_MS = 15_000
109+
const CHUNKDB_RECLAIM_DRAIN_PAUSE_MS = 10
110+
const CHUNKDB_RECLAIM_DRAIN_SLEEP_MS = 250
101111

102112
// A Database layer can be materialized more than once inside one server process
103113
// (for example by independently-built service graphs). Starting one infinite
@@ -171,14 +181,91 @@ interface SealerOptions {
171181
* statement. On DBs without INCREMENTAL auto_vacuum (existing files) this is a
172182
* cheap no-op. Must run OUTSIDE a transaction.
173183
*/
174-
function reclaimSpace(db: DatabaseShape): Effect.Effect<void> {
184+
type ReclaimMode = "maintenance" | "drain"
185+
186+
interface ReclaimStats {
187+
readonly attempted: boolean
188+
readonly reclaimedPages: number
189+
readonly freePages: number
190+
readonly pageCount: number
191+
readonly needsMore: boolean
192+
}
193+
194+
export function shouldDrainFreelist(freePages: number, pageCount: number): boolean {
195+
if (freePages < CHUNKDB_RECLAIM_DRAIN_MIN_FREE_PAGES || pageCount <= 0) return false
196+
return freePages / pageCount > CHUNKDB_RECLAIM_DRAIN_FREE_RATIO
197+
}
198+
199+
function reclaimSpace(db: DatabaseShape, mode: ReclaimMode = "maintenance"): Effect.Effect<ReclaimStats> {
175200
return Effect.gen(function* () {
176201
const rows = yield* db.all<{ auto_vacuum: number }>(`PRAGMA auto_vacuum`).pipe(Effect.orDie)
177-
if ((rows[0]?.auto_vacuum ?? 0) !== 2) return
178-
for (let i = 0; i < CHUNKDB_VACUUM_MAX_ITERATIONS; i++) {
179-
const free = yield* db.all<{ freelist_count: number }>(`PRAGMA freelist_count`).pipe(Effect.orDie)
180-
if ((free[0]?.freelist_count ?? 0) === 0) break
181-
yield* db.run(`PRAGMA incremental_vacuum(${CHUNKDB_VACUUM_PAGES_PER_PASS})`).pipe(Effect.orDie)
202+
if ((rows[0]?.auto_vacuum ?? 0) !== 2) {
203+
return { attempted: false, reclaimedPages: 0, freePages: 0, pageCount: 0, needsMore: false }
204+
}
205+
206+
const initialFreeRows = yield* db.all<{ freelist_count: number }>(`PRAGMA freelist_count`).pipe(Effect.orDie)
207+
const initialPageRows = yield* db.all<{ page_count: number }>(`PRAGMA page_count`).pipe(Effect.orDie)
208+
const initialFree = initialFreeRows[0]?.freelist_count ?? 0
209+
const initialPageCount = initialPageRows[0]?.page_count ?? 0
210+
if (initialFree === 0) {
211+
return { attempted: true, reclaimedPages: 0, freePages: 0, pageCount: initialPageCount, needsMore: false }
212+
}
213+
214+
const started = Date.now()
215+
const maxIterations = mode === "maintenance" ? CHUNKDB_VACUUM_MAX_ITERATIONS : Number.POSITIVE_INFINITY
216+
let iterations = 0
217+
let currentFree = initialFree
218+
let currentPageCount = initialPageCount
219+
220+
while (iterations < maxIterations) {
221+
if (mode === "drain") {
222+
if (!shouldDrainFreelist(currentFree, currentPageCount)) break
223+
if (Date.now() - started >= CHUNKDB_RECLAIM_DRAIN_MAX_MS) break
224+
} else if (currentFree === 0) {
225+
break
226+
}
227+
228+
const vacuum = yield* db.run(`PRAGMA incremental_vacuum(${CHUNKDB_VACUUM_PAGES_PER_PASS})`).pipe(
229+
Effect.map(() => true),
230+
Effect.catch((error) => {
231+
if (isSqliteBusy(error)) return Effect.succeed(false)
232+
return Effect.die(error)
233+
}),
234+
)
235+
if (!vacuum) {
236+
// Foreground owns the single SQLite writer slot. Background reclaim does
237+
// not queue behind it; yield immediately and let the next drain slice try.
238+
break
239+
}
240+
241+
iterations += 1
242+
const freeRows = yield* db.all<{ freelist_count: number }>(`PRAGMA freelist_count`).pipe(Effect.orDie)
243+
const pageRows = yield* db.all<{ page_count: number }>(`PRAGMA page_count`).pipe(Effect.orDie)
244+
const nextFree = freeRows[0]?.freelist_count ?? 0
245+
const nextPageCount = pageRows[0]?.page_count ?? currentPageCount
246+
247+
// Defensive escape hatch: a driver/SQLite combination that reports no
248+
// reclaim progress must never turn the background loop into a CPU spin.
249+
if (nextFree >= currentFree && nextPageCount >= currentPageCount) {
250+
currentFree = nextFree
251+
currentPageCount = nextPageCount
252+
break
253+
}
254+
255+
currentFree = nextFree
256+
currentPageCount = nextPageCount
257+
yield* Effect.yieldNow
258+
if (mode === "drain") yield* Effect.sleep(Duration.millis(CHUNKDB_RECLAIM_DRAIN_PAUSE_MS))
259+
}
260+
261+
const needsMore = mode === "drain" && shouldDrainFreelist(currentFree, currentPageCount)
262+
263+
return {
264+
attempted: true,
265+
reclaimedPages: Math.max(0, initialFree - currentFree),
266+
freePages: currentFree,
267+
pageCount: currentPageCount,
268+
needsMore,
182269
}
183270
})
184271
}
@@ -555,6 +642,13 @@ export function runPassV2(
555642
const frameByID = new Map(compressed)
556643

557644
const plans: Plan[] = []
645+
// Delta matching is useful for modest record-shaped values, but it is the
646+
// wrong representation for jumbo rows. Besides adding read amplification,
647+
// building a correction requires indexing/comparing both raw values. Keep
648+
// jumbo payloads on the independently-decodable v4 segmented frame path.
649+
// This also makes a corrupted/hostile giant event incapable of monopolizing
650+
// a background sealing pass when delta mode is explicitly enabled.
651+
const deltaMaxRawBytes = 4 * 1024 * 1024
558652
for (const plan of pending) {
559653
if (plan.kind === "skip") {
560654
plans.push(plan)
@@ -573,9 +667,13 @@ export function runPassV2(
573667

574668
const frame = frameByID.get(candidate.id) ?? candidate.data
575669
let finalFrame: string | Uint8Array = frame
576-
if (Flag.OPENCODE_SEAL_DELTA && typeof frame !== "string") {
670+
if (
671+
Flag.OPENCODE_SEAL_DELTA &&
672+
typeof frame !== "string" &&
673+
rawBytes.byteLength <= deltaMaxRawBytes
674+
) {
577675
const base = lastBaseByAggregate.get(candidate.aggregate_id)
578-
if (base !== undefined) {
676+
if (base !== undefined && base.raw.byteLength <= deltaMaxRawBytes) {
579677
const { codec, level } = chooseCodec(rawBytes.byteLength)
580678
const delta = compressDeltaRef(rawBytes, base.raw, base.valueId, codec, level)
581679
if (delta.byteLength < frame.byteLength * 0.7) finalFrame = delta
@@ -868,6 +966,7 @@ export function runSealerLoop(filename: string): Effect.Effect<void> {
868966
hotTailEvents: CHUNKDB_HOT_TAIL_EVENTS,
869967
})
870968
let previousHitCap: boolean = false
969+
let reclaimDraining = false
871970
let backoffMs = BACKOFF_BASE_MS
872971
for (;;) {
873972
const draining: boolean = previousHitCap && backfillAllowed
@@ -914,7 +1013,36 @@ export function runSealerLoop(filename: string): Effect.Effect<void> {
9141013
)
9151014
yield* Effect.logDebug("ChunkDB sealer idle", { filename, ...backlog })
9161015
}
917-
const wait = previousHitCap && backfillAllowed ? DRAIN_SLEEP_MS : MAINTENANCE_INTERVAL_MS
1016+
1017+
// When sealing no longer has a capped backlog, immediately inspect
1018+
// the physical freelist. A multi-GB historical backfill can leave the
1019+
// logical DB tiny while SQLite still owns gigabytes of free pages.
1020+
// Drain those pages in low-priority 15s slices until the free-space
1021+
// ratio is sane instead of waiting ten minutes between ~160MB
1022+
// maintenance reclaims. This also repairs a DB that finished its
1023+
// backfill in a previous process before this policy existed.
1024+
if (!previousHitCap) {
1025+
const reclaim = yield* reclaimSpace(db, "drain")
1026+
reclaimDraining = reclaim.needsMore
1027+
if (reclaim.reclaimedPages > 0) {
1028+
yield* Effect.logInfo("ChunkDB space reclaim progress", {
1029+
filename,
1030+
reclaimedPages: reclaim.reclaimedPages,
1031+
freePages: reclaim.freePages,
1032+
pageCount: reclaim.pageCount,
1033+
freeRatio: reclaim.pageCount > 0 ? reclaim.freePages / reclaim.pageCount : 0,
1034+
draining: reclaim.needsMore,
1035+
})
1036+
}
1037+
} else {
1038+
reclaimDraining = false
1039+
}
1040+
1041+
const wait = previousHitCap && backfillAllowed
1042+
? DRAIN_SLEEP_MS
1043+
: reclaimDraining
1044+
? CHUNKDB_RECLAIM_DRAIN_SLEEP_MS
1045+
: MAINTENANCE_INTERVAL_MS
9181046
yield* Effect.sleep(Duration.millis(wait))
9191047
}
9201048
}),

‎packages/core/src/database/json-codec.ts‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -418,6 +418,13 @@ const DELTA_TAG_COPY = 0x01
418418
const DELTA_MIN_COPY = 4
419419
const DELTA_MAX_COPY = 65535
420420
const DELTA_MAX_LITERAL = 65535
421+
// A 4-byte key can occur millions of times in repetitive JSON/diff payloads.
422+
// Keeping every occurrence turns the matcher below from its intended bounded
423+
// average-case behavior into catastrophic O(n^2) work and unbounded index
424+
// memory. Retain a small rolling sample of recent positions per key instead.
425+
// 64 candidates is enough to preserve useful long-span matches while placing a
426+
// hard ceiling on work performed at each new-value position.
427+
const DELTA_MAX_MATCH_CANDIDATES = 64
421428

422429
/** True when `bytes` is an OCDB v5 (delta_ref) frame. */
423430
export function isV5Frame(bytes: Uint8Array): boolean {
@@ -440,7 +447,8 @@ export function encodeV5Correction(base: Uint8Array, newValue: Uint8Array): Uint
440447
list = []
441448
index.set(w, list)
442449
}
443-
list.push(p)
450+
if (list.length < DELTA_MAX_MATCH_CANDIDATES) list.push(p)
451+
else list[p & (DELTA_MAX_MATCH_CANDIDATES - 1)] = p
444452
}
445453
const out: number[] = []
446454
let lit: number[] = []

‎packages/core/test/database/chunkdb-crash.test.ts‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ import { Database as CoreDatabase, Service as DatabaseService, withBackfillDb }
3232
import type { DatabaseShape } from "../../src/database/database"
3333
import { DatabaseMigration } from "../../src/database/migration"
3434
import { CHUNKDB_COOLING_MS, CHUNKDB_HOT_TAIL_EVENTS, ensureChunkDB } from "../../src/database/chunkdb"
35-
import { inspectSealerBacklog, runPassV2 } from "../../src/database/chunk-sealer"
35+
import { inspectSealerBacklog, runPassV2, shouldDrainFreelist } from "../../src/database/chunk-sealer"
3636
import { rehydrateEvents, CdbRehydrateError } from "../../src/event"
3737
import { EventV2 } from "../../src/event"
3838
import { Event } from "@opencode-ai/schema/event"
@@ -179,6 +179,15 @@ function tmpDb() {
179179
}
180180

181181
describe("ChunkDB crash recovery", () => {
182+
test("RECLAIM POLICY: large freelists stay in accelerated drain while normal slack does not", () => {
183+
// Production incident shape: ~813k free pages out of ~1.265M total must
184+
// remain in accelerated reclaim rather than falling back to 10-minute idle.
185+
expect(shouldDrainFreelist(813_377, 1_264_872)).toBe(true)
186+
// Stop once physical slack is <=5% or too small to justify churn.
187+
expect(shouldDrainFreelist(50_000, 1_100_000)).toBe(false)
188+
expect(shouldDrainFreelist(1_000, 1_000_000)).toBe(false)
189+
})
190+
182191
test("FOREGROUND PRIORITY: backfill yields on SQLITE_BUSY and resumes after the writer lock is released", async () => {
183192
const { dir, path } = tmpDb()
184193
let locker: BunDatabase | undefined

‎packages/core/test/database/json-codec-delta.test.ts‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,19 @@ describe("ChunkDB epoch-4 v5 delta_ref frame", () => {
8080
expect(JSON.parse(decoder.decode(reconstructed))).toEqual(newObj)
8181
})
8282

83+
test("highly repetitive inputs stay bounded and round-trip", () => {
84+
// Regression: retaining every position for a common 4-byte key made the
85+
// matcher effectively O(n^2) on large repetitive JSON/diff payloads. This
86+
// shape deliberately creates hundreds of thousands of identical windows.
87+
const baseRaw = encoder.encode(`{"text":"${"A".repeat(512 * 1024)}"}`)
88+
const newRaw = encoder.encode(`{"text":"${"A".repeat(512 * 1024 - 1)}B"}`)
89+
const v5 = compressDeltaRef(newRaw, baseRaw, "agg:repetitive", 1, 1)
90+
const header = parseV5Header(v5)
91+
const correction = decodeV5Correction(header.correction, header.codec, header.storedCrc)
92+
const reconstructed = applyV5Correction(baseRaw, correction, header.totalRawLen)
93+
expect(reconstructed).toEqual(newRaw)
94+
})
95+
8396
test("fail-closed on corrupt correction CRC", () => {
8497
const baseRaw = encoder.encode(JSON.stringify(makeBase()))
8598
const newRaw = encoder.encode(JSON.stringify(makeNew()))

0 commit comments

Comments
 (0)