Skip to content

Commit c573bd9

Browse files
authored
fix(storage): harden #558 satellite backup writes and zst restore (#571)
* fix(storage): harden #558 satellite backup writes and zst restore Make every satellite-backup.json update crash-safe (tmp + fsync + rename + best-effort dir fsync) so a failed post-memories rewrite cannot truncate the last valid snapshot. Restore legacy compressed-only quarantine entries by decompressing a lone .jsonl.zst in memory and reconstructing the thread row. * fix(storage): harden satellite-backup write retries for #558 follow-up Loop short writes to completion and rename via renameAtomicFile so Windows sharing violations cannot replace a durable temp with a truncated backup.
1 parent c0b902f commit c573bd9

5 files changed

Lines changed: 363 additions & 24 deletions

File tree

‎src/codex/history-provider.ts‎

Lines changed: 33 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,18 @@
11
import { createHash } from "node:crypto";
22
import { closeSync, existsSync, fsyncSync, mkdirSync, openSync, readFileSync, readSync, unlinkSync, writeSync } from "node:fs";
33
import { dirname, join, resolve } from "node:path";
4+
import { zstdDecompressSync } from "node:zlib";
45
import { Database } from "bun:sqlite";
56
import { CODEX_HOME } from "./paths";
67
import { atomicWriteFile, getConfigDir } from "../config";
78

9+
/**
10+
* Cap for decompressing a lone `.jsonl.zst` rollout during quarantine restore.
11+
* Bounds peak memory while reconstructing thread rows; never write the decoded
12+
* JSONL to disk.
13+
*/
14+
export const MAX_ROLLOUT_ZST_DECOMPRESSED_BYTES = 64 * 1024 * 1024;
15+
816
const STATE_DB_PATH = join(CODEX_HOME, "state_5.sqlite");
917
function historyBackupPathFor(stateDbPath: string): string {
1018
const normalized = process.platform === "win32" ? resolve(stateDbPath).toLowerCase() : resolve(stateDbPath);
@@ -332,15 +340,39 @@ function extractUserMessagePreview(line: string): string | null {
332340
/**
333341
* Reconstruct thread identity + listing fields from a staged/restored rollout JSONL.
334342
* Returns null when the file is missing or has no parseable `session_meta`.
343+
*
344+
* Accepts plain `.jsonl` or a lone `.jsonl.zst` (legacy Phase-2 quarantine). Compressed
345+
* rollouts are decompressed in memory with {@link MAX_ROLLOUT_ZST_DECOMPRESSED_BYTES};
346+
* no decompressed copy is written to disk.
335347
*/
336348
export function readThreadFieldsFromRollout(path: string): RolloutThreadFields | null {
337349
if (!path || !existsSync(path)) return null;
338350
let raw: string;
339351
try {
340-
raw = readFileSync(path, "utf8");
352+
raw = path.endsWith(".zst")
353+
? decompressRolloutZstUtf8(path)
354+
: readFileSync(path, "utf8");
341355
} catch {
342356
return null;
343357
}
358+
return parseThreadFieldsFromRolloutText(raw);
359+
}
360+
361+
function decompressRolloutZstUtf8(
362+
path: string,
363+
maxBytes: number = MAX_ROLLOUT_ZST_DECOMPRESSED_BYTES,
364+
): string {
365+
const compressed = readFileSync(path);
366+
const decoded = zstdDecompressSync(compressed as Uint8Array<ArrayBuffer>, {
367+
maxOutputLength: maxBytes,
368+
});
369+
if (decoded.byteLength > maxBytes) {
370+
throw new Error("rollout_zst_too_large");
371+
}
372+
return new TextDecoder().decode(decoded);
373+
}
374+
375+
function parseThreadFieldsFromRolloutText(raw: string): RolloutThreadFields | null {
344376
const lines = raw.split("\n");
345377
let latest: ParsedSessionMeta | null = null;
346378
let firstUserMessage = "";

‎src/server/management/logs-usage-routes.ts‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -362,13 +362,12 @@ export async function handleLogsUsageRoutes(ctx: ManagementContext): Promise<Res
362362
}
363363
}
364364

365-
if (
366-
process.env.OPENCODEX_CLEANUP_TEST_HOOKS === "1" &&
367-
url.pathname === "/api/storage/trash/restore/test-stream" &&
368-
req.method === "GET"
369-
) {
370-
const stream = getRestoreTrashTestStreamResponse();
371-
if (stream) return stream;
365+
if (url.pathname === "/api/storage/trash/restore/test-stream" && req.method === "GET") {
366+
if (process.env.OPENCODEX_CLEANUP_TEST_HOOKS === "1") {
367+
const stream = getRestoreTrashTestStreamResponse();
368+
if (stream) return stream;
369+
}
370+
// Always answer this test-only path — never fall through to the GUI SPA (200 HTML).
372371
return jsonResponse({ error: "not_available" }, 404);
373372
}
374373

‎src/storage/cleanup.ts‎

Lines changed: 99 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ import { basename, isAbsolute, join, relative, resolve, sep } from "node:path";
3434
import { Database } from "bun:sqlite";
3535
import { resolveCodexHomeDir } from "../codex/home";
3636
import { readThreadFieldsFromRollout } from "../codex/history-provider";
37+
import { renameAtomicFile } from "../config";
3738

3839
export const ARCHIVED_SESSIONS_DIR = "archived_sessions";
3940
export const TRASH_DIR = ".trash";
@@ -739,13 +740,20 @@ interface ReconcileTestHooks {
739740
failBeforeStateCommit?: boolean;
740741
failSatelliteRestore?: boolean;
741742
failSatelliteBackupWrite?: boolean;
743+
/**
744+
* Fail a satellite-backup.json *replacement* after the temp is durable but before
745+
* rename — exercises crash-safety of the post-memories rewrite without truncating
746+
* the last valid backup.
747+
*/
748+
failSatelliteBackupReplace?: boolean;
742749
/** Runs after satellite deletes are committed, before state thread deletion. */
743750
afterSatelliteMutations?: () => void;
744751
}
745752

746753
const SATELLITE_BACKUP_FILE = "satellite-backup.json";
747754
/** Marks an incomplete restore so retries can accept dest files and resume metadata. */
748755
const RESTORE_PENDING_FILE = "restore-pending.json";
756+
let _satelliteBackupSeq = 0;
749757

750758
type StagedFile = { from: string; to: string; relPath: string };
751759

@@ -971,8 +979,66 @@ function restoreConsolidateGlobalJob(
971979
}
972980
}
973981

974-
function writeSatelliteBackup(stageDir: string, backup: SatelliteBackup): void {
975-
writePrivateFile(join(stageDir, SATELLITE_BACKUP_FILE), JSON.stringify(backup));
982+
/**
983+
* Best-effort directory fsync so a preceding rename is durable on crash.
984+
* Unsupported on some Windows setups — never treat failure as fatal.
985+
*/
986+
function fsyncDirectoryBestEffort(dirPath: string): void {
987+
let fd: number | undefined;
988+
try {
989+
fd = openSync(dirPath, "r");
990+
fsyncSync(fd);
991+
} catch {
992+
/* best-effort */
993+
} finally {
994+
if (fd !== undefined) {
995+
try { closeSync(fd); } catch { /* */ }
996+
}
997+
}
998+
}
999+
1000+
/**
1001+
* Atomically replace satellite-backup.json: private temp in the stage, full write + fsync,
1002+
* rename (with Windows sharing-violation retries), then best-effort directory fsync.
1003+
* An interrupted update never truncates the last valid backup that was written before a
1004+
* satellite DB commit.
1005+
*/
1006+
function writeSatelliteBackup(
1007+
stageDir: string,
1008+
backup: SatelliteBackup,
1009+
options?: { failWrite?: boolean; failReplaceBeforeRename?: boolean },
1010+
): void {
1011+
if (options?.failWrite) throw new Error("test_fail_satellite_backup_write");
1012+
const dest = join(stageDir, SATELLITE_BACKUP_FILE);
1013+
const replacing = existsSync(dest);
1014+
const tmp = join(stageDir, `${SATELLITE_BACKUP_FILE}.${process.pid}.${++_satelliteBackupSeq}.tmp`);
1015+
const payload = Buffer.from(JSON.stringify(backup), "utf8");
1016+
const fd = openSync(tmp, "w", 0o600);
1017+
try {
1018+
let offset = 0;
1019+
while (offset < payload.length) {
1020+
offset += writeSync(fd, payload, offset, payload.length - offset, null);
1021+
}
1022+
fsyncSync(fd);
1023+
} catch (error) {
1024+
try { closeSync(fd); } catch { /* */ }
1025+
try { unlinkSync(tmp); } catch { /* */ }
1026+
throw error;
1027+
}
1028+
closeSync(fd);
1029+
chmodPrivatePath(tmp, 0o600);
1030+
if (options?.failReplaceBeforeRename && replacing) {
1031+
try { unlinkSync(tmp); } catch { /* */ }
1032+
throw new Error("test_fail_satellite_backup_replace");
1033+
}
1034+
try {
1035+
renameAtomicFile(tmp, dest);
1036+
} catch (error) {
1037+
try { unlinkSync(tmp); } catch { /* */ }
1038+
throw error;
1039+
}
1040+
chmodPrivatePath(dest, 0o600);
1041+
fsyncDirectoryBestEffort(stageDir);
9761042
}
9771043

9781044
function clearSatelliteBackup(stageDir: string): void {
@@ -1240,15 +1306,17 @@ function deleteAndCommitSatellites(
12401306
deleteMemoriesInTx(locks.memories.db, backup.memories);
12411307
if (backup.memories.consolidateTouched) {
12421308
// Capture under the write lock, but persist only after COMMIT+close.
1243-
// Holding BEGIN IMMEDIATE across writeFileSync lets Windows CI disk/AV
1244-
// latency stall the lock long enough for concurrent reopen hooks (and
1245-
// bun's default 5s test timeout) to hang — see PR #558 windows-latest.
1309+
// Holding BEGIN IMMEDIATE across a durable backup rewrite lets Windows CI
1310+
// disk/AV latency stall the lock long enough for concurrent reopen hooks
1311+
// (and bun's default 5s test timeout) to hang — see PR #558 windows-latest.
12461312
backup.memories.consolidatePostImage = readConsolidateGlobalJob(locks.memories.db);
12471313
}
12481314
commitSatelliteLock(locks.memories);
12491315
locks.memories = undefined;
12501316
if (backup.memories.consolidateTouched) {
1251-
writeSatelliteBackup(stageDir, backup);
1317+
writeSatelliteBackup(stageDir, backup, {
1318+
failReplaceBeforeRename: hooks?.failSatelliteBackupReplace,
1319+
});
12521320
}
12531321
if (hooks?.failAfterMemoriesMutation) throw new Error("test_fail_after_memories");
12541322
}
@@ -1420,10 +1488,9 @@ function reconcileDeletedThreads(
14201488
backup.dynamicTools = stateDeps.dynamicTools;
14211489
backup.spawnEdges = stateDeps.spawnEdges;
14221490
try {
1423-
if (hooks?.failSatelliteBackupWrite) {
1424-
throw new Error("test_fail_satellite_backup_write");
1425-
}
1426-
writeSatelliteBackup(stageDir, backup);
1491+
writeSatelliteBackup(stageDir, backup, {
1492+
failWrite: hooks?.failSatelliteBackupWrite,
1493+
});
14271494
} catch {
14281495
rollbackAllSatelliteLocks(satelliteLocks);
14291496
stateDb.exec("ROLLBACK");
@@ -1606,6 +1673,7 @@ export interface ExecuteCleanupOptions {
16061673
failBeforeStateCommit?: boolean;
16071674
failSatelliteRestore?: boolean;
16081675
failSatelliteBackupWrite?: boolean;
1676+
failSatelliteBackupReplace?: boolean;
16091677
afterSatelliteMutations?: () => void;
16101678
};
16111679
}
@@ -1635,6 +1703,7 @@ export function pickWireCleanupTestHooks(raw: unknown): CleanupWireTestHooks | u
16351703
if (typeof o.failBeforeStateCommit === "boolean") out.failBeforeStateCommit = o.failBeforeStateCommit;
16361704
if (typeof o.failSatelliteRestore === "boolean") out.failSatelliteRestore = o.failSatelliteRestore;
16371705
if (typeof o.failSatelliteBackupWrite === "boolean") out.failSatelliteBackupWrite = o.failSatelliteBackupWrite;
1706+
if (typeof o.failSatelliteBackupReplace === "boolean") out.failSatelliteBackupReplace = o.failSatelliteBackupReplace;
16381707
return Object.keys(out).length > 0 ? out : undefined;
16391708
}
16401709

@@ -2350,16 +2419,30 @@ function restoreThreadsFromManifest(
23502419
);
23512420
const reconstructed: SqlRow[] = [];
23522421
for (const entry of toReconstruct) {
2353-
let abs: string;
2422+
let abs: string | undefined;
23542423
try {
23552424
abs = absFromRel(codexHome, entry.rolloutPath!);
23562425
} catch {
2357-
// Prefer the first restored physical path under archived_sessions.
2358-
const rel = entry.physicalRelPaths[0];
2359-
if (!rel) throw new Error("missing_rollout_for_thread");
2360-
abs = absFromRel(codexHome, rel);
2426+
abs = undefined;
2427+
}
2428+
// Legacy compressed-only quarantine: manifest rolloutPath is often the logical
2429+
// `.jsonl` name while the only restored physical file is `.jsonl.zst`.
2430+
if (!abs || !existsSync(abs)) {
2431+
for (const rel of entry.physicalRelPaths) {
2432+
try {
2433+
const candidate = absFromRel(codexHome, rel);
2434+
if (existsSync(candidate)) {
2435+
abs = candidate;
2436+
break;
2437+
}
2438+
} catch {
2439+
/* try next physical path */
2440+
}
2441+
}
23612442
}
2362-
// .jsonl.zst cannot be parsed here — require a plain .jsonl sibling or path.
2443+
if (!abs) throw new Error("missing_rollout_for_thread");
2444+
// Prefer a plain .jsonl sibling when present; otherwise readThreadFieldsFromRollout
2445+
// decompresses a lone .jsonl.zst in memory (bounded) for legacy quarantine restores.
23632446
if (abs.endsWith(ZST_SUFFIX)) {
23642447
const plain = abs.slice(0, -".zst".length);
23652448
if (existsSync(plain)) abs = plain;

0 commit comments

Comments
 (0)