Skip to content

Commit ce72f5c

Browse files
committed
fix: make journal updates serialized and atomic per checkpoint
Implement the pre-merge Persistence Integrity demand: appendChanges replaces the per-entry append with one serialized (per-task) atomic update - read current journal, repair a torn tail, write the complete next content to a temp file, fsync it, atomically rename onto changes.jsonl, and fsync the parent directory (POSIX; Windows cannot open directories). A crash leaves either the old or the new complete journal, never a half-applied multi-file update. checkpointSave now persists the whole step's writes in that single update and propagates journal failures instead of reporting a fully durable checkpoint with lost entries; the change card is emitted only after the journal is durable. Save failures keep the historical handler (logged, checkpoints disabled, resolves undefined). Regression tests cover post-commit failure, mid-update rename failure (all-or-nothing, temp cleanup), concurrent-update serialization, restart recovery from a stale temp file and torn tail, and empty updates.
1 parent edeb407 commit ce72f5c

4 files changed

Lines changed: 234 additions & 118 deletions

File tree

‎src/core/checkpoints/__tests__/changeJournal.spec.ts‎

Lines changed: 66 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import path from "path"
44

55
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"
66

7-
import { appendChange, journalPath, loadChanges, type ChangeJournalEntry } from "../changeJournal"
7+
import { appendChange, appendChanges, journalPath, loadChanges, type ChangeJournalEntry } from "../changeJournal"
88

99
describe("changeJournal", () => {
1010
const taskId = "test-task"
@@ -72,7 +72,7 @@ describe("changeJournal", () => {
7272
expect(raw.split("\n").filter((line) => line !== "")).toHaveLength(2)
7373
})
7474

75-
it("syncs the parent directory after the first append creates the journal (POSIX)", async () => {
75+
it("syncs the parent directory after every journal replacement (POSIX)", async () => {
7676
const platformDesc = Object.getOwnPropertyDescriptor(process, "platform")
7777
Object.defineProperty(process, "platform", { value: "linux", configurable: true })
7878
const dirPath = path.dirname(journalPath(tmpRoot, taskId))
@@ -100,9 +100,10 @@ describe("changeJournal", () => {
100100
await appendChange(tmpRoot, taskId, entry({ checkpointId: "first" }))
101101
expect(synced).toEqual([dirPath])
102102
expect(closed).toEqual([dirPath])
103-
// Subsequent appends do not re-sync the directory.
103+
// Every replacement renames a new file into place, so the directory
104+
// entry is re-synced on each update.
104105
await appendChange(tmpRoot, taskId, entry({ checkpointId: "second" }))
105-
expect(synced).toHaveLength(1)
106+
expect(synced).toHaveLength(2)
106107
} finally {
107108
openSpy.mockRestore()
108109
if (platformDesc) Object.defineProperty(process, "platform", platformDesc)
@@ -111,22 +112,23 @@ describe("changeJournal", () => {
111112
expect(result.map((e) => e.checkpointId)).toEqual(["first", "second"])
112113
})
113114

114-
it("propagates a non-ENOENT tail-inspection failure instead of appending blind", async () => {
115+
it("propagates a non-ENOENT journal-read failure instead of replacing blind", async () => {
115116
await appendChange(tmpRoot, taskId, entry({ checkpointId: "good" }))
116117
const filePath = journalPath(tmpRoot, taskId)
117118
const raw = await fs.readFile(filePath, "utf8")
118119
await fs.writeFile(filePath, raw + '{"path":"src/half.ts","operation":"upd')
119-
// The tail cannot be inspected (I/O error): appending would merge the
120-
// new entry into the fragment, so appendChange must fail loudly.
121-
const statSpy = vi
122-
.spyOn(fs, "stat")
120+
// The current content cannot be read (I/O error): replacing would
121+
// discard whatever the unreadable journal really held, so appendChange
122+
// must fail loudly.
123+
const readSpy = vi
124+
.spyOn(fs, "readFile")
123125
.mockRejectedValueOnce(Object.assign(new Error("simulated I/O error"), { code: "EIO" }))
124126
try {
125127
await expect(appendChange(tmpRoot, taskId, entry({ checkpointId: "after" }))).rejects.toMatchObject({
126128
code: "EIO",
127129
})
128130
} finally {
129-
statSpy.mockRestore()
131+
readSpy.mockRestore()
130132
}
131133
// The failed append wrote nothing: the next append still repairs and loads.
132134
await appendChange(tmpRoot, taskId, entry({ checkpointId: "after" }))
@@ -140,7 +142,7 @@ describe("changeJournal", () => {
140142
const openSpy = vi.spyOn(fs, "open").mockImplementation(async (pathish, flags) => {
141143
const openAs = realOpen as (p: string | URL, f?: string) => Promise<fs.FileHandle>
142144
const handle = await openAs(pathish as string | URL, flags as string | undefined)
143-
if (flags === "a") {
145+
if (flags === "w") {
144146
const realWrite = handle.write.bind(handle)
145147
let first = true
146148
handle.write = (async (buf: Uint8Array, off?: number, len?: number) => {
@@ -163,6 +165,59 @@ describe("changeJournal", () => {
163165
const result = await loadChanges(tmpRoot, taskId)
164166
expect(result.map((e) => e.checkpointId)).toEqual(["first", "second"])
165167
})
168+
169+
it("serializes concurrent updates to the same task", async () => {
170+
await Promise.all([
171+
appendChange(tmpRoot, taskId, entry({ checkpointId: "c1" })),
172+
appendChange(tmpRoot, taskId, entry({ checkpointId: "c2" })),
173+
])
174+
175+
const result = await loadChanges(tmpRoot, taskId)
176+
expect(result.map((e) => e.checkpointId).sort()).toEqual(["c1", "c2"])
177+
})
178+
179+
it("leaves the journal untouched when the atomic rename fails mid-update", async () => {
180+
await appendChanges(tmpRoot, taskId, [entry({ checkpointId: "kept" })])
181+
const renameSpy = vi.spyOn(fs, "rename").mockRejectedValueOnce(new Error("simulated rename failure"))
182+
try {
183+
await expect(
184+
appendChanges(tmpRoot, taskId, [
185+
entry({ checkpointId: "m1", path: "src/m1.ts" }),
186+
entry({ checkpointId: "m2", path: "src/m2.ts" }),
187+
]),
188+
).rejects.toThrow("simulated rename failure")
189+
} finally {
190+
renameSpy.mockRestore()
191+
}
192+
// Neither of the two new entries survived: the update is all-or-nothing.
193+
const result = await loadChanges(tmpRoot, taskId)
194+
expect(result.map((e) => e.checkpointId)).toEqual(["kept"])
195+
// The failed temp file was cleaned up.
196+
const leftovers = (await fs.readdir(path.dirname(journalPath(tmpRoot, taskId)))).filter((name) =>
197+
name.endsWith(".tmp"),
198+
)
199+
expect(leftovers).toEqual([])
200+
})
201+
202+
it("recovers after a crash left a stale temp file and a torn tail", async () => {
203+
await appendChange(tmpRoot, taskId, entry({ checkpointId: "before" }))
204+
const dir = path.dirname(journalPath(tmpRoot, taskId))
205+
// A crash between the temp write and the rename leaves an orphan temp
206+
// file; a crash mid-write can also leave the tail unterminated.
207+
await fs.writeFile(path.join(dir, "changes.jsonl.4242.orphan.tmp"), "partial")
208+
const raw = await fs.readFile(journalPath(tmpRoot, taskId), "utf8")
209+
await fs.writeFile(journalPath(tmpRoot, taskId), raw + '{"path":"src/half.ts","operation":"upd')
210+
211+
await appendChange(tmpRoot, taskId, entry({ checkpointId: "after" }))
212+
213+
const result = await loadChanges(tmpRoot, taskId)
214+
expect(result.map((e) => e.checkpointId)).toEqual(["before", "after"])
215+
})
216+
217+
it("does not create a journal for an empty update", async () => {
218+
await appendChanges(tmpRoot, taskId, [])
219+
await expect(fs.stat(journalPath(tmpRoot, taskId))).rejects.toMatchObject({ code: "ENOENT" })
220+
})
166221
})
167222

168223
describe("task ID containment", () => {

‎src/core/checkpoints/__tests__/checkpointJournal.test.ts‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -197,23 +197,27 @@ describe("checkpointSave change-journal wiring (B2)", () => {
197197
expect(await loadChanges(tmpStorageDir, TASK_ID)).toEqual([])
198198
})
199199

200-
it("logs and continues when the journal cannot be written (checkpoints stay enabled)", async () => {
201-
// Block the per-task checkpoint dir so the journal mkdir/append fails.
200+
it("propagates a journal persistence failure after the commit and emits no change card", async () => {
201+
// Block the per-task checkpoint dir so the journal mkdir/read fails.
202202
const taskDir = path.join(tmpStorageDir, "tasks", TASK_ID)
203203
await fs.mkdir(taskDir, { recursive: true })
204204
await fs.writeFile(path.join(taskDir, "checkpoints"), "blocker")
205205

206206
const consoleErrorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined)
207207

208-
// The journal failure is swallowed (logged, not rethrown), so the
209-
// checkpoint result still resolves exactly as without journaling.
210-
await expect(checkpointSave(mockTask as Task, false, true, write)).resolves.toMatchObject({ commit: COMMIT })
208+
// The checkpoint committed but the journal update did not persist: the
209+
// save must not report a fully durable checkpoint, and the change card
210+
// (which advertises the journaled files) must not be emitted.
211+
await expect(checkpointSave(mockTask as Task, false, true, write)).rejects.toThrow()
211212

213+
// Propagation is not a checkpoint error: checkpoints stay enabled.
212214
expect(mockTask.enableCheckpoints).toBe(true)
213215
expect(consoleErrorSpy).toHaveBeenCalledWith(
214-
expect.stringContaining("failed to append change journal entry"),
216+
expect.stringContaining("failed to persist change journal entries"),
215217
expect.anything(),
216218
)
219+
const cardCalls = vi.mocked(mockTask.say).mock.calls.filter((call) => call[0] === "change_card")
220+
expect(cardCalls).toHaveLength(0)
217221
})
218222
})
219223

‎src/core/checkpoints/changeJournal.ts‎

Lines changed: 103 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -48,69 +48,118 @@ export function journalPath(globalStorageDir: string, taskId: string): string {
4848
}
4949

5050
/**
51-
* Append one change-journal entry to the per-task changes.jsonl file.
51+
* Per-task update queue: journal replacements for one task never overlap,
52+
* so concurrent checkpoint saves cannot lose each other's entries through
53+
* the read-modify-write cycle below.
54+
*/
55+
const journalQueues = new Map<string, Promise<unknown>>()
56+
57+
function withTaskLock<T>(key: string, run: () => Promise<T>): Promise<T> {
58+
const prev = journalQueues.get(key) ?? Promise.resolve()
59+
// The next update runs after the previous one settles, whatever it did.
60+
const next = prev.then(run, run)
61+
// The queued promise swallows its outcome so a failed update neither
62+
// blocks later updates nor surfaces as an unhandled rejection; callers
63+
// observe the failure through `next`.
64+
journalQueues.set(
65+
key,
66+
next.then(
67+
() => undefined,
68+
() => undefined,
69+
),
70+
)
71+
return next
72+
}
73+
74+
/**
75+
* Append change-journal entries to the per-task changes.jsonl file as one
76+
* serialized, atomic update.
77+
*
78+
* The whole multi-entry update is a crash-safe replacement: the current
79+
* journal is read, a torn tail is terminated, the complete next content is
80+
* written to a temporary file in the same directory and fsynced, then
81+
* atomically renamed onto the journal, and the parent directory is fsynced
82+
* (POSIX; Windows cannot open directories). A crash at any point leaves
83+
* either the old complete journal or the new complete journal - never a
84+
* half-applied multi-file update - and a durable checkpoint implies a
85+
* durable journal entry. Creates parent directories if they don't exist
86+
* yet (e.g. first checkpoint).
5287
*
53-
* Torn-tail repair: a crash mid-append can leave the file without a trailing
54-
* newline; appending directly would splice the new entry into the fragment and
55-
* lose both at load time, so the tail is terminated before the new line is
56-
* written. The append is fsynced so a durable checkpoint implies a durable
57-
* journal entry; a crash mid-write can still tear the appended line itself,
58-
* which loadChanges tolerates as a torn tail. Creates parent directories if
59-
* they don't exist yet (e.g. first checkpoint).
88+
* Failures propagate: callers must not report a fully durable checkpoint
89+
* when the journal update did not complete.
6090
*/
61-
export async function appendChange(globalStorageDir: string, taskId: string, entry: ChangeJournalEntry): Promise<void> {
91+
export async function appendChanges(
92+
globalStorageDir: string,
93+
taskId: string,
94+
entries: ChangeJournalEntry[],
95+
): Promise<void> {
96+
if (entries.length === 0) {
97+
return
98+
}
6299
const filePath = journalPath(globalStorageDir, taskId)
63-
await fs.mkdir(path.dirname(filePath), { recursive: true })
64-
let payload = JSON.stringify(entry) + "\n"
65-
let created = false
66-
try {
67-
const stat = await fs.stat(filePath)
68-
if (stat.size > 0) {
69-
const tail = await fs.open(filePath, "r")
100+
return withTaskLock(filePath, async () => {
101+
const dir = path.dirname(filePath)
102+
await fs.mkdir(dir, { recursive: true })
103+
let content = ""
104+
try {
105+
content = await fs.readFile(filePath, "utf8")
106+
} catch (error) {
107+
// A missing journal is a fresh history. Any other read failure
108+
// means the complete next content cannot be built, so propagate
109+
// instead of replacing the journal with a partial view of it.
110+
if ((error as NodeJS.ErrnoException).code !== "ENOENT") {
111+
throw error
112+
}
113+
}
114+
let payload = entries.map((entry) => JSON.stringify(entry) + "\n").join("")
115+
// Torn-tail repair: a journal left without a trailing newline (e.g.
116+
// a crash before this replacement path completed) must be terminated
117+
// before the new lines or the first appended entry merges into the
118+
// fragment.
119+
if (content && !content.endsWith("\n")) {
120+
payload = "\n" + payload
121+
}
122+
const tmpPath = path.join(dir, `${JOURNAL_FILENAME}.${process.pid}.${Math.random().toString(36).slice(2)}.tmp`)
123+
const data = Buffer.from(content + payload)
124+
try {
125+
const handle = await fs.open(tmpPath, "w")
70126
try {
71-
const last = Buffer.alloc(1)
72-
await tail.read(last, 0, 1, stat.size - 1)
73-
if (last[0] !== 0x0a) {
74-
payload = "\n" + payload
127+
// A single write can complete partially; sync only after the
128+
// whole content is on the handle, or a short write would
129+
// rename a torn journal into place.
130+
let written = 0
131+
while (written < data.length) {
132+
const res = await handle.write(data, written, data.length - written)
133+
written += res.bytesWritten
75134
}
135+
await handle.sync()
76136
} finally {
77-
await tail.close()
137+
await handle.close()
78138
}
79-
}
80-
} catch (error) {
81-
// A missing journal is a fresh append. Any other inspection failure
82-
// means we cannot know whether a separator is required, so propagate
83-
// instead of appending blind and losing the entry to a merged line.
84-
if ((error as NodeJS.ErrnoException).code !== "ENOENT") {
139+
await fs.rename(tmpPath, filePath)
140+
if (process.platform !== "win32") {
141+
// The rename replaced the directory entry: on POSIX the
142+
// directory itself must be synced for the new entry to
143+
// survive a crash (Windows cannot open directories).
144+
const dirHandle = await fs.open(dir, "r")
145+
try {
146+
await dirHandle.sync()
147+
} finally {
148+
await dirHandle.close()
149+
}
150+
}
151+
} catch (error) {
152+
// Best-effort cleanup: the rename never completed, so the journal
153+
// still holds the previous complete content; drop the temp file.
154+
await fs.rm(tmpPath, { force: true }).catch(() => {})
85155
throw error
86156
}
87-
created = true
88-
}
89-
const data = Buffer.from(payload)
90-
const handle = await fs.open(filePath, "a")
91-
try {
92-
// A single write can complete partially; sync only after the whole
93-
// record is on the handle, or a short write would persist a torn entry
94-
// while appendChange reports success.
95-
let written = 0
96-
while (written < data.length) {
97-
const res = await handle.write(data, written, data.length - written)
98-
written += res.bytesWritten
99-
}
100-
await handle.sync()
101-
} finally {
102-
await handle.close()
103-
}
104-
if (created && process.platform !== "win32") {
105-
// The append created the file: on POSIX the new directory entry needs
106-
// its own sync to survive a crash (Windows cannot open directories).
107-
const dir = await fs.open(path.dirname(filePath), "r")
108-
try {
109-
await dir.sync()
110-
} finally {
111-
await dir.close()
112-
}
113-
}
157+
})
158+
}
159+
160+
/** Append a single entry; see appendChanges for the durability contract. */
161+
export async function appendChange(globalStorageDir: string, taskId: string, entry: ChangeJournalEntry): Promise<void> {
162+
await appendChanges(globalStorageDir, taskId, [entry])
114163
}
115164

116165
/**

0 commit comments

Comments
 (0)