Skip to content

Commit b11dede

Browse files
committed
fix(storage): harden #558 satellite backup writes and zst restore
Make every satellite-backup.json update crash-safe (tmp + fsync + rename + best-effort dir fsync) so a failed post-memories rewrite cannot truncate the last valid snapshot. Restore legacy compressed-only quarantine entries by decompressing a lone .jsonl.zst in memory and reconstructing the thread row.
1 parent 40fdc3d commit b11dede

4 files changed

Lines changed: 356 additions & 24 deletions

File tree

‎src/codex/history-provider.ts‎

Lines changed: 33 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,18 @@
11
import { createHash } from "node:crypto";
22
import { closeSync, existsSync, fsyncSync, mkdirSync, openSync, readFileSync, readSync, unlinkSync, writeSync } from "node:fs";
33
import { dirname, join, resolve } from "node:path";
4+
import { zstdDecompressSync } from "node:zlib";
45
import { Database } from "bun:sqlite";
56
import { CODEX_HOME } from "./paths";
67
import { atomicWriteFile, getConfigDir } from "../config";
78

9+
/**
10+
* Cap for decompressing a lone `.jsonl.zst` rollout during quarantine restore.
11+
* Bounds peak memory while reconstructing thread rows; never write the decoded
12+
* JSONL to disk.
13+
*/
14+
export const MAX_ROLLOUT_ZST_DECOMPRESSED_BYTES = 64 * 1024 * 1024;
15+
816
const STATE_DB_PATH = join(CODEX_HOME, "state_5.sqlite");
917
function historyBackupPathFor(stateDbPath: string): string {
1018
const normalized = process.platform === "win32" ? resolve(stateDbPath).toLowerCase() : resolve(stateDbPath);
@@ -332,15 +340,39 @@ function extractUserMessagePreview(line: string): string | null {
332340
/**
333341
* Reconstruct thread identity + listing fields from a staged/restored rollout JSONL.
334342
* Returns null when the file is missing or has no parseable `session_meta`.
343+
*
344+
* Accepts plain `.jsonl` or a lone `.jsonl.zst` (legacy Phase-2 quarantine). Compressed
345+
* rollouts are decompressed in memory with {@link MAX_ROLLOUT_ZST_DECOMPRESSED_BYTES};
346+
* no decompressed copy is written to disk.
335347
*/
336348
export function readThreadFieldsFromRollout(path: string): RolloutThreadFields | null {
337349
if (!path || !existsSync(path)) return null;
338350
let raw: string;
339351
try {
340-
raw = readFileSync(path, "utf8");
352+
raw = path.endsWith(".zst")
353+
? decompressRolloutZstUtf8(path)
354+
: readFileSync(path, "utf8");
341355
} catch {
342356
return null;
343357
}
358+
return parseThreadFieldsFromRolloutText(raw);
359+
}
360+
361+
function decompressRolloutZstUtf8(
362+
path: string,
363+
maxBytes: number = MAX_ROLLOUT_ZST_DECOMPRESSED_BYTES,
364+
): string {
365+
const compressed = readFileSync(path);
366+
const decoded = zstdDecompressSync(compressed as Uint8Array<ArrayBuffer>, {
367+
maxOutputLength: maxBytes,
368+
});
369+
if (decoded.byteLength > maxBytes) {
370+
throw new Error("rollout_zst_too_large");
371+
}
372+
return new TextDecoder().decode(decoded);
373+
}
374+
375+
function parseThreadFieldsFromRolloutText(raw: string): RolloutThreadFields | null {
344376
const lines = raw.split("\n");
345377
let latest: ParsedSessionMeta | null = null;
346378
let firstUserMessage = "";

‎src/server/management/logs-usage-routes.ts‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -362,13 +362,12 @@ export async function handleLogsUsageRoutes(ctx: ManagementContext): Promise<Res
362362
}
363363
}
364364

365-
if (
366-
process.env.OPENCODEX_CLEANUP_TEST_HOOKS === "1" &&
367-
url.pathname === "/api/storage/trash/restore/test-stream" &&
368-
req.method === "GET"
369-
) {
370-
const stream = getRestoreTrashTestStreamResponse();
371-
if (stream) return stream;
365+
if (url.pathname === "/api/storage/trash/restore/test-stream" && req.method === "GET") {
366+
if (process.env.OPENCODEX_CLEANUP_TEST_HOOKS === "1") {
367+
const stream = getRestoreTrashTestStreamResponse();
368+
if (stream) return stream;
369+
}
370+
// Always answer this test-only path — never fall through to the GUI SPA (200 HTML).
372371
return jsonResponse({ error: "not_available" }, 404);
373372
}
374373

‎src/storage/cleanup.ts‎

Lines changed: 94 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -739,13 +739,20 @@ interface ReconcileTestHooks {
739739
failBeforeStateCommit?: boolean;
740740
failSatelliteRestore?: boolean;
741741
failSatelliteBackupWrite?: boolean;
742+
/**
743+
* Fail a satellite-backup.json *replacement* after the temp is durable but before
744+
* rename — exercises crash-safety of the post-memories rewrite without truncating
745+
* the last valid backup.
746+
*/
747+
failSatelliteBackupReplace?: boolean;
742748
/** Runs after satellite deletes are committed, before state thread deletion. */
743749
afterSatelliteMutations?: () => void;
744750
}
745751

746752
const SATELLITE_BACKUP_FILE = "satellite-backup.json";
747753
/** Marks an incomplete restore so retries can accept dest files and resume metadata. */
748754
const RESTORE_PENDING_FILE = "restore-pending.json";
755+
let _satelliteBackupSeq = 0;
749756

750757
type StagedFile = { from: string; to: string; relPath: string };
751758

@@ -971,8 +978,62 @@ function restoreConsolidateGlobalJob(
971978
}
972979
}
973980

974-
function writeSatelliteBackup(stageDir: string, backup: SatelliteBackup): void {
975-
writePrivateFile(join(stageDir, SATELLITE_BACKUP_FILE), JSON.stringify(backup));
981+
/**
982+
* Best-effort directory fsync so a preceding rename is durable on crash.
983+
* Unsupported on some Windows setups — never treat failure as fatal.
984+
*/
985+
function fsyncDirectoryBestEffort(dirPath: string): void {
986+
let fd: number | undefined;
987+
try {
988+
fd = openSync(dirPath, "r");
989+
fsyncSync(fd);
990+
} catch {
991+
/* best-effort */
992+
} finally {
993+
if (fd !== undefined) {
994+
try { closeSync(fd); } catch { /* */ }
995+
}
996+
}
997+
}
998+
999+
/**
1000+
* Atomically replace satellite-backup.json: private temp in the stage, fsync, rename,
1001+
* then best-effort directory fsync. An interrupted update never truncates the last
1002+
* valid backup that was written before a satellite DB commit.
1003+
*/
1004+
function writeSatelliteBackup(
1005+
stageDir: string,
1006+
backup: SatelliteBackup,
1007+
options?: { failWrite?: boolean; failReplaceBeforeRename?: boolean },
1008+
): void {
1009+
if (options?.failWrite) throw new Error("test_fail_satellite_backup_write");
1010+
const dest = join(stageDir, SATELLITE_BACKUP_FILE);
1011+
const replacing = existsSync(dest);
1012+
const tmp = join(stageDir, `${SATELLITE_BACKUP_FILE}.${process.pid}.${++_satelliteBackupSeq}.tmp`);
1013+
const payload = JSON.stringify(backup);
1014+
const fd = openSync(tmp, "w", 0o600);
1015+
try {
1016+
writeSync(fd, payload, null, "utf8");
1017+
fsyncSync(fd);
1018+
} catch (error) {
1019+
try { closeSync(fd); } catch { /* */ }
1020+
try { unlinkSync(tmp); } catch { /* */ }
1021+
throw error;
1022+
}
1023+
closeSync(fd);
1024+
chmodPrivatePath(tmp, 0o600);
1025+
if (options?.failReplaceBeforeRename && replacing) {
1026+
try { unlinkSync(tmp); } catch { /* */ }
1027+
throw new Error("test_fail_satellite_backup_replace");
1028+
}
1029+
try {
1030+
renameSync(tmp, dest);
1031+
} catch (error) {
1032+
try { unlinkSync(tmp); } catch { /* */ }
1033+
throw error;
1034+
}
1035+
chmodPrivatePath(dest, 0o600);
1036+
fsyncDirectoryBestEffort(stageDir);
9761037
}
9771038

9781039
function clearSatelliteBackup(stageDir: string): void {
@@ -1240,15 +1301,17 @@ function deleteAndCommitSatellites(
12401301
deleteMemoriesInTx(locks.memories.db, backup.memories);
12411302
if (backup.memories.consolidateTouched) {
12421303
// Capture under the write lock, but persist only after COMMIT+close.
1243-
// Holding BEGIN IMMEDIATE across writeFileSync lets Windows CI disk/AV
1244-
// latency stall the lock long enough for concurrent reopen hooks (and
1245-
// bun's default 5s test timeout) to hang — see PR #558 windows-latest.
1304+
// Holding BEGIN IMMEDIATE across a durable backup rewrite lets Windows CI
1305+
// disk/AV latency stall the lock long enough for concurrent reopen hooks
1306+
// (and bun's default 5s test timeout) to hang — see PR #558 windows-latest.
12461307
backup.memories.consolidatePostImage = readConsolidateGlobalJob(locks.memories.db);
12471308
}
12481309
commitSatelliteLock(locks.memories);
12491310
locks.memories = undefined;
12501311
if (backup.memories.consolidateTouched) {
1251-
writeSatelliteBackup(stageDir, backup);
1312+
writeSatelliteBackup(stageDir, backup, {
1313+
failReplaceBeforeRename: hooks?.failSatelliteBackupReplace,
1314+
});
12521315
}
12531316
if (hooks?.failAfterMemoriesMutation) throw new Error("test_fail_after_memories");
12541317
}
@@ -1420,10 +1483,9 @@ function reconcileDeletedThreads(
14201483
backup.dynamicTools = stateDeps.dynamicTools;
14211484
backup.spawnEdges = stateDeps.spawnEdges;
14221485
try {
1423-
if (hooks?.failSatelliteBackupWrite) {
1424-
throw new Error("test_fail_satellite_backup_write");
1425-
}
1426-
writeSatelliteBackup(stageDir, backup);
1486+
writeSatelliteBackup(stageDir, backup, {
1487+
failWrite: hooks?.failSatelliteBackupWrite,
1488+
});
14271489
} catch {
14281490
rollbackAllSatelliteLocks(satelliteLocks);
14291491
stateDb.exec("ROLLBACK");
@@ -1606,6 +1668,7 @@ export interface ExecuteCleanupOptions {
16061668
failBeforeStateCommit?: boolean;
16071669
failSatelliteRestore?: boolean;
16081670
failSatelliteBackupWrite?: boolean;
1671+
failSatelliteBackupReplace?: boolean;
16091672
afterSatelliteMutations?: () => void;
16101673
};
16111674
}
@@ -1635,6 +1698,7 @@ export function pickWireCleanupTestHooks(raw: unknown): CleanupWireTestHooks | u
16351698
if (typeof o.failBeforeStateCommit === "boolean") out.failBeforeStateCommit = o.failBeforeStateCommit;
16361699
if (typeof o.failSatelliteRestore === "boolean") out.failSatelliteRestore = o.failSatelliteRestore;
16371700
if (typeof o.failSatelliteBackupWrite === "boolean") out.failSatelliteBackupWrite = o.failSatelliteBackupWrite;
1701+
if (typeof o.failSatelliteBackupReplace === "boolean") out.failSatelliteBackupReplace = o.failSatelliteBackupReplace;
16381702
return Object.keys(out).length > 0 ? out : undefined;
16391703
}
16401704

@@ -2350,16 +2414,30 @@ function restoreThreadsFromManifest(
23502414
);
23512415
const reconstructed: SqlRow[] = [];
23522416
for (const entry of toReconstruct) {
2353-
let abs: string;
2417+
let abs: string | undefined;
23542418
try {
23552419
abs = absFromRel(codexHome, entry.rolloutPath!);
23562420
} catch {
2357-
// Prefer the first restored physical path under archived_sessions.
2358-
const rel = entry.physicalRelPaths[0];
2359-
if (!rel) throw new Error("missing_rollout_for_thread");
2360-
abs = absFromRel(codexHome, rel);
2421+
abs = undefined;
2422+
}
2423+
// Legacy compressed-only quarantine: manifest rolloutPath is often the logical
2424+
// `.jsonl` name while the only restored physical file is `.jsonl.zst`.
2425+
if (!abs || !existsSync(abs)) {
2426+
for (const rel of entry.physicalRelPaths) {
2427+
try {
2428+
const candidate = absFromRel(codexHome, rel);
2429+
if (existsSync(candidate)) {
2430+
abs = candidate;
2431+
break;
2432+
}
2433+
} catch {
2434+
/* try next physical path */
2435+
}
2436+
}
23612437
}
2362-
// .jsonl.zst cannot be parsed here — require a plain .jsonl sibling or path.
2438+
if (!abs) throw new Error("missing_rollout_for_thread");
2439+
// Prefer a plain .jsonl sibling when present; otherwise readThreadFieldsFromRollout
2440+
// decompresses a lone .jsonl.zst in memory (bounded) for legacy quarantine restores.
23632441
if (abs.endsWith(ZST_SUFFIX)) {
23642442
const plain = abs.slice(0, -".zst".length);
23652443
if (existsSync(plain)) abs = plain;

0 commit comments

Comments
 (0)