Skip to content

Commit 69b60e5

Browse files
committed
fix: make journal updates serialized and atomic per checkpoint
Implement the pre-merge Persistence Integrity demand: appendChanges replaces the per-entry append with one serialized (per-task) atomic update - read current journal, repair a torn tail, write the complete next content to a temp file, fsync it, atomically rename onto changes.jsonl, and fsync the parent directory (POSIX; Windows cannot open directories). A crash leaves either the old or the new complete journal, never a half-applied multi-file update. checkpointSave now persists the whole step's writes in that single update and propagates journal failures instead of reporting a fully durable checkpoint with lost entries; the change card is emitted only after the journal is durable. Save failures keep the historical handler (logged, checkpoints disabled, resolves undefined). Regression tests cover post-commit failure, mid-update rename failure (all-or-nothing, temp cleanup), concurrent-update serialization, restart recovery from a stale temp file and torn tail, and empty updates.
1 parent 993e740 commit 69b60e5

4 files changed

Lines changed: 324 additions & 105 deletions

File tree

‎src/core/checkpoints/__tests__/changeJournal.spec.ts‎

Lines changed: 102 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import path from "path"
44

55
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"
66

7-
import { appendChange, journalPath, loadChanges, type ChangeJournalEntry } from "../changeJournal"
7+
import { appendChange, appendChanges, journalPath, loadChanges, type ChangeJournalEntry } from "../changeJournal"
88

99
describe("changeJournal", () => {
1010
const taskId = "test-task"
@@ -72,22 +72,63 @@ describe("changeJournal", () => {
7272
expect(raw.split("\n").filter((line) => line !== "")).toHaveLength(2)
7373
})
7474

75-
it("propagates a non-ENOENT tail-inspection failure instead of appending blind", async () => {
75+
it("syncs the parent directory after every journal replacement (POSIX)", async () => {
76+
const platformDesc = Object.getOwnPropertyDescriptor(process, "platform")
77+
Object.defineProperty(process, "platform", { value: "linux", configurable: true })
78+
const dirPath = path.dirname(journalPath(tmpRoot, taskId))
79+
const realOpen = fs.open
80+
const synced: string[] = []
81+
const closed: string[] = []
82+
// Windows cannot open directories, so the directory handle is doubled
83+
// here: the code under test only opens it on non-win32 platforms.
84+
const openSpy = vi.spyOn(fs, "open").mockImplementation(async (pathish, flags) => {
85+
const openAs = realOpen as (p: string | URL, f?: string) => Promise<fs.FileHandle>
86+
const p = typeof pathish === "string" ? pathish : String(pathish)
87+
if (p === dirPath && flags === "r") {
88+
return {
89+
sync: async () => {
90+
synced.push(p)
91+
},
92+
close: async () => {
93+
closed.push(p)
94+
},
95+
} as unknown as fs.FileHandle
96+
}
97+
return openAs(p, flags as string | undefined)
98+
})
99+
try {
100+
await appendChange(tmpRoot, taskId, entry({ checkpointId: "first" }))
101+
expect(synced).toEqual([dirPath])
102+
expect(closed).toEqual([dirPath])
103+
// Every replacement renames a new file into place, so the directory
104+
// entry is re-synced on each update.
105+
await appendChange(tmpRoot, taskId, entry({ checkpointId: "second" }))
106+
expect(synced).toHaveLength(2)
107+
} finally {
108+
openSpy.mockRestore()
109+
if (platformDesc) Object.defineProperty(process, "platform", platformDesc)
110+
}
111+
const result = await loadChanges(tmpRoot, taskId)
112+
expect(result.map((e) => e.checkpointId)).toEqual(["first", "second"])
113+
})
114+
115+
it("propagates a non-ENOENT journal-read failure instead of replacing blind", async () => {
76116
await appendChange(tmpRoot, taskId, entry({ checkpointId: "good" }))
77117
const filePath = journalPath(tmpRoot, taskId)
78118
const raw = await fs.readFile(filePath, "utf8")
79119
await fs.writeFile(filePath, raw + '{"path":"src/half.ts","operation":"upd')
80-
// The tail cannot be inspected (I/O error): appending would merge the
81-
// new entry into the fragment, so appendChange must fail loudly.
82-
const statSpy = vi
83-
.spyOn(fs, "stat")
120+
// The current content cannot be read (I/O error): replacing would
121+
// discard whatever the unreadable journal really held, so appendChange
122+
// must fail loudly.
123+
const readSpy = vi
124+
.spyOn(fs, "readFile")
84125
.mockRejectedValueOnce(Object.assign(new Error("simulated I/O error"), { code: "EIO" }))
85126
try {
86127
await expect(appendChange(tmpRoot, taskId, entry({ checkpointId: "after" }))).rejects.toMatchObject({
87128
code: "EIO",
88129
})
89130
} finally {
90-
statSpy.mockRestore()
131+
readSpy.mockRestore()
91132
}
92133
// The failed append wrote nothing: the next append still repairs and loads.
93134
await appendChange(tmpRoot, taskId, entry({ checkpointId: "after" }))
@@ -101,7 +142,7 @@ describe("changeJournal", () => {
101142
const openSpy = vi.spyOn(fs, "open").mockImplementation(async (pathish, flags) => {
102143
const openAs = realOpen as (p: string | URL, f?: string) => Promise<fs.FileHandle>
103144
const handle = await openAs(pathish as string | URL, flags as string | undefined)
104-
if (flags === "a") {
145+
if (flags === "w") {
105146
const realWrite = handle.write.bind(handle)
106147
let first = true
107148
handle.write = (async (buf: Uint8Array, off?: number, len?: number) => {
@@ -124,6 +165,59 @@ describe("changeJournal", () => {
124165
const result = await loadChanges(tmpRoot, taskId)
125166
expect(result.map((e) => e.checkpointId)).toEqual(["first", "second"])
126167
})
168+
169+
it("serializes concurrent updates to the same task", async () => {
170+
await Promise.all([
171+
appendChange(tmpRoot, taskId, entry({ checkpointId: "c1" })),
172+
appendChange(tmpRoot, taskId, entry({ checkpointId: "c2" })),
173+
])
174+
175+
const result = await loadChanges(tmpRoot, taskId)
176+
expect(result.map((e) => e.checkpointId).sort()).toEqual(["c1", "c2"])
177+
})
178+
179+
it("leaves the journal untouched when the atomic rename fails mid-update", async () => {
180+
await appendChanges(tmpRoot, taskId, [entry({ checkpointId: "kept" })])
181+
const renameSpy = vi.spyOn(fs, "rename").mockRejectedValueOnce(new Error("simulated rename failure"))
182+
try {
183+
await expect(
184+
appendChanges(tmpRoot, taskId, [
185+
entry({ checkpointId: "m1", path: "src/m1.ts" }),
186+
entry({ checkpointId: "m2", path: "src/m2.ts" }),
187+
]),
188+
).rejects.toThrow("simulated rename failure")
189+
} finally {
190+
renameSpy.mockRestore()
191+
}
192+
// Neither of the two new entries survived: the update is all-or-nothing.
193+
const result = await loadChanges(tmpRoot, taskId)
194+
expect(result.map((e) => e.checkpointId)).toEqual(["kept"])
195+
// The failed temp file was cleaned up.
196+
const leftovers = (await fs.readdir(path.dirname(journalPath(tmpRoot, taskId)))).filter((name) =>
197+
name.endsWith(".tmp"),
198+
)
199+
expect(leftovers).toEqual([])
200+
})
201+
202+
it("recovers after a crash left a stale temp file and a torn tail", async () => {
203+
await appendChange(tmpRoot, taskId, entry({ checkpointId: "before" }))
204+
const dir = path.dirname(journalPath(tmpRoot, taskId))
205+
// A crash between the temp write and the rename leaves an orphan temp
206+
// file; a crash mid-write can also leave the tail unterminated.
207+
await fs.writeFile(path.join(dir, "changes.jsonl.4242.orphan.tmp"), "partial")
208+
const raw = await fs.readFile(journalPath(tmpRoot, taskId), "utf8")
209+
await fs.writeFile(journalPath(tmpRoot, taskId), raw + '{"path":"src/half.ts","operation":"upd')
210+
211+
await appendChange(tmpRoot, taskId, entry({ checkpointId: "after" }))
212+
213+
const result = await loadChanges(tmpRoot, taskId)
214+
expect(result.map((e) => e.checkpointId)).toEqual(["before", "after"])
215+
})
216+
217+
it("does not create a journal for an empty update", async () => {
218+
await appendChanges(tmpRoot, taskId, [])
219+
await expect(fs.stat(journalPath(tmpRoot, taskId))).rejects.toMatchObject({ code: "ENOENT" })
220+
})
127221
})
128222

129223
describe("task ID containment", () => {

‎src/core/checkpoints/__tests__/checkpointJournal.test.ts‎

Lines changed: 46 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -197,22 +197,62 @@ describe("checkpointSave change-journal wiring (B2)", () => {
197197
expect(await loadChanges(tmpStorageDir, TASK_ID)).toEqual([])
198198
})
199199

200-
it("logs and continues when the journal cannot be written (checkpoints stay enabled)", async () => {
201-
// Block the per-task checkpoint dir so the journal mkdir/append fails.
200+
it("propagates a journal persistence failure after the commit and emits no change card", async () => {
201+
// Block the per-task checkpoint dir so the journal mkdir/read fails.
202202
const taskDir = path.join(tmpStorageDir, "tasks", TASK_ID)
203203
await fs.mkdir(taskDir, { recursive: true })
204204
await fs.writeFile(path.join(taskDir, "checkpoints"), "blocker")
205205

206206
const consoleErrorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined)
207207

208-
// The journal failure is swallowed (logged, not rethrown), so the
209-
// checkpoint result still resolves exactly as without journaling.
210-
await expect(checkpointSave(mockTask as Task, false, true, write)).resolves.toMatchObject({ commit: COMMIT })
208+
// The checkpoint committed but the journal update did not persist: the
209+
// save must not report a fully durable checkpoint, and the change card
210+
// (which advertises the journaled files) must not be emitted.
211+
await expect(checkpointSave(mockTask as Task, false, true, write)).rejects.toThrow()
211212

213+
// Propagation is not a checkpoint error: checkpoints stay enabled.
212214
expect(mockTask.enableCheckpoints).toBe(true)
213215
expect(consoleErrorSpy).toHaveBeenCalledWith(
214-
expect.stringContaining("failed to append change journal entry"),
216+
expect.stringContaining("failed to persist change journal entries"),
215217
expect.anything(),
216218
)
219+
const cardCalls = vi.mocked(mockTask.say).mock.calls.filter((call) => call[0] === "change_card")
220+
expect(cardCalls).toHaveLength(0)
221+
})
222+
})
223+
224+
describe("loadChanges entry-shape validation (B2)", () => {
225+
let tmpStorageDir: string
226+
227+
beforeEach(async () => {
228+
tmpStorageDir = await fs.mkdtemp(path.join(os.tmpdir(), "b2-journal-shape-"))
229+
})
230+
231+
afterEach(async () => {
232+
await fs.rm(tmpStorageDir, { recursive: true, force: true })
233+
})
234+
235+
it("skips lines that are valid JSON but not entry-shaped", async () => {
236+
const dir = path.join(tmpStorageDir, "tasks", TASK_ID, "checkpoints")
237+
await fs.mkdir(dir, { recursive: true })
238+
const valid = JSON.stringify({ path: "src/a.ts", operation: "create", checkpointId: "c1" })
239+
await fs.writeFile(
240+
path.join(dir, "changes.jsonl"),
241+
[
242+
// A torn append can leave valid JSON that is not an entry.
243+
"1",
244+
"null",
245+
// Missing checkpointId.
246+
JSON.stringify({ path: "src/b.ts", operation: "update" }),
247+
// Unknown operation.
248+
JSON.stringify({ path: "src/c.ts", operation: "rename", checkpointId: "c2" }),
249+
// Non-string path.
250+
JSON.stringify({ path: 42, operation: "create", checkpointId: "c3" }),
251+
valid,
252+
].join("\n") + "\n",
253+
)
254+
255+
const entries = await loadChanges(tmpStorageDir, TASK_ID)
256+
expect(entries).toEqual([{ path: "src/a.ts", operation: "create", checkpointId: "c1" }])
217257
})
218258
})

0 commit comments

Comments
 (0)