Skip to content

Commit 5c6be04

Browse files
committed
fix(lab): finalize projection statements so a rebuild can replace its own file
rebuildLabProjection closed its database without finalizing the statements it had prepared. Bun keeps a prepared statement alive until it is finalized or collected, and on Windows an outstanding statement holds the file open: `close()` leaves the handle behind and `close(true)` throws "database is locked". The next rebuild then could not unlink the projection it was replacing, and the retry loop in wipeSqlite could only convert that into a slower failure -- "failed to remove stale projection file after retries". POSIX permits unlinking an open file, which is why a rebuild that is deterministic by contract was only ever non-deterministic on Windows. Collect the prepared statements and finalize them before the close. This is the real defect behind ten Compatibility Lab failures across the ledger, fabric-task and public-evidence suites, all of which called rebuild more than once. Two server tests also exceeded Bun's 5s default while binding real proxies: the Retry-After case runs two full pool-passthrough cycles and the #702 case binds one proxy per route class to prove none of them reaches upstream. In both the servers are the assertion, so they take the existing SERVER_BUDGET_MS rather than a new knob.
1 parent c388221 commit 5c6be04

3 files changed

Lines changed: 46 additions & 20 deletions

File tree

‎src/lab/projection/rebuild.ts‎

Lines changed: 36 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -122,57 +122,66 @@ export function rebuildLabProjection(configDir?: string): RebuildResult {
122122

123123
const db = new Database(paths.sqlitePath);
124124
let transactionOpen = false;
125+
// Every statement prepared below, so they can be finalized before the close.
126+
//
127+
// Bun keeps a prepared statement alive until it is finalized or garbage collected,
128+
// and on Windows an unfinalized statement holds the database file open: `close()`
129+
// leaves the handle, and `close(true)` throws "database is locked". A second
130+
// rebuild then failed to unlink the previous projection with EBUSY, and the retry
131+
// loop in `wipeSqlite` could only turn that into a slower failure. POSIX allows
132+
// unlinking an open file, which is why this never surfaced there.
133+
const prepared: Array<{ finalize(): void }> = [];
134+
const prepare = (sql: string) => {
135+
const statement = db.prepare(sql);
136+
prepared.push(statement);
137+
return statement;
138+
};
125139
try {
126140
db.exec("PRAGMA journal_mode=DELETE;");
127141
db.exec("PRAGMA foreign_keys=OFF;");
128142
db.exec("BEGIN IMMEDIATE;");
129143
transactionOpen = true;
130144
resetProjectionSchema(db);
131145

132-
db.prepare(
133-
"INSERT OR REPLACE INTO schema_meta(key, value) VALUES (?, ?)",
134-
).run("schema_version", String(LAB_SQLITE_SCHEMA_VERSION));
135-
db.prepare(
136-
"INSERT OR REPLACE INTO schema_meta(key, value) VALUES (?, ?)",
137-
).run("projection_spec_version", LAB_PROJECTION_SPEC_VERSION);
138-
db.prepare(
139-
"INSERT OR REPLACE INTO schema_meta(key, value) VALUES (?, ?)",
140-
).run("built_at_ms", String(Date.now()));
146+
const insertMeta = prepare("INSERT OR REPLACE INTO schema_meta(key, value) VALUES (?, ?)");
147+
insertMeta.run("schema_version", String(LAB_SQLITE_SCHEMA_VERSION));
148+
insertMeta.run("projection_spec_version", LAB_PROJECTION_SPEC_VERSION);
149+
insertMeta.run("built_at_ms", String(Date.now()));
141150

142-
const insertCorruption = db.prepare(
151+
const insertCorruption = prepare(
143152
"INSERT INTO corruption(kind, line_number, event_id, detail) VALUES (?, ?, ?, ?)",
144153
);
145154
for (const c of corruptions) {
146155
insertCorruption.run(c.kind, c.lineNumber ?? null, c.eventId ?? null, c.detail);
147156
}
148157

149-
const insertEvent = db.prepare(
158+
const insertEvent = prepare(
150159
`INSERT INTO events(event_id, event_kind, recorded_at, producer, producer_version, payload_json, excluded, exclusion_reason)
151160
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
152161
);
153-
const insertSubject = db.prepare(
162+
const insertSubject = prepare(
154163
`INSERT OR IGNORE INTO subjects(subject_id, subject_kind, subject_json) VALUES (?, ?, ?)`,
155164
);
156-
const insertObs = db.prepare(
165+
const insertObs = prepare(
157166
`INSERT INTO observations(
158167
event_id, subject_id, evidence_layer, suite_id, suite_version, suite_manifest_digest,
159168
scenario_id, scenario_version, scenario_manifest_digest, outcome, completed_at, execution_mode
160169
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
161170
);
162-
const insertClaim = db.prepare(
171+
const insertClaim = prepare(
163172
`INSERT INTO claims(
164173
event_id, subject_id, capability, polarity, source_manifest_digest,
165174
effective_at, recorded_at, supersedes_json, current, usable
166175
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
167176
);
168-
const insertInv = db.prepare(
177+
const insertInv = prepare(
169178
`INSERT INTO invalidations(event_id, reason, targets_json, recorded_at, applied) VALUES (?, ?, ?, ?, ?)`,
170179
);
171-
const insertPurge = db.prepare(
180+
const insertPurge = prepare(
172181
`INSERT INTO purges(event_id, target_event_ids_json, target_artifact_digests_json, purge_actions_json, recorded_at)
173182
VALUES (?, ?, ?, ?, ?)`,
174183
);
175-
const insertArtifact = db.prepare(
184+
const insertArtifact = prepare(
176185
`INSERT INTO artifacts(digest, artifact_class, media_type, byte_count, status, last_error)
177186
VALUES (?, ?, ?, ?, ?, ?)
178187
ON CONFLICT(digest) DO UPDATE SET
@@ -319,7 +328,7 @@ export function rebuildLabProjection(configDir?: string): RebuildResult {
319328
}
320329
}
321330

322-
const insertVerdict = db.prepare(
331+
const insertVerdict = prepare(
323332
`INSERT INTO verdicts(
324333
projection_key, subject_id, evidence_layer, suite_id, suite_version, suite_manifest_digest,
325334
projection_spec_version, verdict, as_of, scenario_manifest_digests_json, claim_source_digest,
@@ -369,6 +378,15 @@ export function rebuildLabProjection(configDir?: string): RebuildResult {
369378
} catch {
370379
// Closing the disposable DB is still safe if pragma restoration fails.
371380
}
381+
// Finalize before closing: an outstanding statement keeps the file open on
382+
// Windows, and the next rebuild cannot unlink the projection it is replacing.
383+
for (const statement of prepared) {
384+
try {
385+
statement.finalize();
386+
} catch {
387+
// A statement already finalized by an error path is not a rebuild failure.
388+
}
389+
}
372390
db.close();
373391
artifactStore.close();
374392
}

‎tests/issue-452-empty-503.test.ts‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import { startServer } from "../src/server";
1313
import { formatPassthroughUpstreamError } from "../src/server/responses/passthrough-error";
1414
import type { OcxConfig, OcxParsedRequest } from "../src/types";
1515
import { installIsolatedCodexHome, type IsolatedCodexHome } from "./helpers/isolated-codex-home";
16+
import { SERVER_BUDGET_MS } from "./helpers/test-budget";
1617

1718
const previousApiToken = process.env.OPENCODEX_API_AUTH_TOKEN;
1819
const previousOpencodexHome = process.env.OPENCODEX_HOME;
@@ -208,7 +209,10 @@ describe("passthrough empty 503 (#452)", () => {
208209
},
209210
);
210211
}
211-
});
212+
// Two full pool-passthrough cycles, each binding a real proxy and a real upstream,
213+
// so the wait is the assertion rather than an accident: it measured ~6s here against
214+
// Bun's 5s default.
215+
}, SERVER_BUDGET_MS);
212216

213217
test("direct /v1/responses drops invalid Retry-After on empty-body 503", async () => {
214218
await withPoolPassthrough(

‎tests/issue-702-expired-replay-state.test.ts‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import { startServer } from "../src/server";
2020
import type { OcxConfig } from "../src/types";
2121
import { fakeChatGptJwt } from "./helpers/fake-chatgpt-jwt";
2222
import { installIsolatedCodexHome, type IsolatedCodexHome } from "./helpers/isolated-codex-home";
23+
import { SERVER_BUDGET_MS } from "./helpers/test-budget";
2324

2425
const originalFetch = globalThis.fetch;
2526
const previousOpencodexHome = process.env.OPENCODEX_HOME;
@@ -334,7 +335,10 @@ describe("Issue #702 expired forward replay state", () => {
334335
} finally {
335336
globalThis.fetch = originalFetch;
336337
}
337-
});
338+
// Three route classes, each binding a real proxy: the per-class servers ARE the
339+
// assertion that no route reaches upstream, and they measured ~6s against Bun's
340+
// 5s default.
341+
}, SERVER_BUDGET_MS);
338342

339343
test("forward mode fails closed when previous response replay state has expired", async () => {
340344
let quotaPrimeCalls = 0;

0 commit comments

Comments
 (0)