Skip to content

Commit 4ddd6ce

Browse files
cliffhallclaude
andcommitted
fix(memory): serialize stale-lock breaking behind a guard file
The staleness check read the lock's contents and its mtime with two separate calls, so they could describe different lock files, and two waiters could both judge one abandoned lock stale, letting one remove the lock the other had just taken. Now the owner and mtime are read through one open handle, and breaking takes a second O_EXCL guard (<lock>.break) and re-checks the lock under it, so only one waiter breaks at a time. A guard left by a crash is cleared after staleMs. Co-Authored-By: Claude Opus 5.5 <[email protected]> Signed-off-by: cliffhall <[email protected]>
1 parent fc7a9be commit 4ddd6ce

2 files changed

Lines changed: 158 additions & 30 deletions

File tree

‎src/memory/__tests__/file-lock.test.ts‎

Lines changed: 90 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -135,32 +135,91 @@ describe("withFileLock", () => {
135135
},
136136
);
137137

138-
it("does not break a stale lock that was replaced before it could be removed", async () => {
139-
await fs.writeFile(lockPath, "stale");
138+
async function writeStaleLock(contents = "stale"): Promise<void> {
139+
await fs.writeFile(lockPath, contents);
140140
const longAgo = new Date(Date.now() - 120_000);
141141
await fs.utimes(lockPath, longAgo, longAgo);
142-
// Another waiter breaks the stale lock and takes it between this waiter's
143-
// first read and its re-check; that waiter's lock must survive.
144-
const realReadFile = fs.readFile;
145-
vi.spyOn(fs, "readFile")
146-
.mockImplementationOnce(realReadFile)
147-
.mockImplementationOnce((async () => {
148-
await fs.writeFile(lockPath, "fresh");
149-
return "fresh";
150-
}) as unknown as typeof fs.readFile);
142+
}
143+
144+
// Run hook once, the first time fs.open is called with these arguments.
145+
function onceOnOpen(
146+
file: string,
147+
flags: string,
148+
hook: () => Promise<void>,
149+
): void {
150+
const realOpen = fs.open;
151+
let fired = false;
152+
vi.spyOn(fs, "open").mockImplementation((async (
153+
...args: Parameters<typeof fs.open>
154+
) => {
155+
if (!fired && args[0] === file && args[1] === flags) {
156+
fired = true;
157+
await hook();
158+
}
159+
return realOpen(...args);
160+
}) as typeof fs.open);
161+
}
162+
163+
it("does not break a lock another waiter replaced while it was judged stale", async () => {
164+
await writeStaleLock();
165+
// Another waiter breaks the stale lock and takes it after this waiter
166+
// judged it stale but before this waiter takes the breaking guard.
167+
onceOnOpen(`${lockPath}.break`, "wx", async () => {
168+
await fs.unlink(lockPath);
169+
await fs.writeFile(lockPath, "fresh");
170+
});
151171

152172
await expect(
153173
withFileLock(lockPath, async () => "ran", fast),
154174
).rejects.toThrow("Timed out");
155175
expect(await fs.readFile(lockPath, "utf-8")).toBe("fresh");
176+
expect(await fs.readdir(dir)).toEqual(["memory.jsonl.lock"]);
177+
});
178+
179+
it("leaves a stale lock to the waiter already breaking it", async () => {
180+
await writeStaleLock();
181+
await fs.writeFile(`${lockPath}.break`, "");
182+
183+
await expect(
184+
withFileLock(lockPath, async () => "ran", fast),
185+
).rejects.toThrow("Timed out");
186+
expect(await fs.readFile(lockPath, "utf-8")).toBe("stale");
187+
});
188+
189+
it("clears a breaking guard left by a crash, then breaks the stale lock", async () => {
190+
await writeStaleLock();
191+
const breakerPath = `${lockPath}.break`;
192+
await fs.writeFile(breakerPath, "");
193+
const longAgo = new Date(Date.now() - 120_000);
194+
await fs.utimes(breakerPath, longAgo, longAgo);
195+
vi.spyOn(console, "error").mockImplementation(() => {});
196+
197+
expect(await withFileLock(lockPath, async () => "ran", fast)).toBe("ran");
198+
expect(await fs.readdir(dir)).toEqual([]);
199+
});
200+
201+
it("fails when the breaking guard cannot be created", async () => {
202+
await writeStaleLock();
203+
const realOpen = fs.open;
204+
vi.spyOn(fs, "open").mockImplementation((async (
205+
...args: Parameters<typeof fs.open>
206+
) => {
207+
if (args[0] === `${lockPath}.break`) {
208+
throw Object.assign(new Error("EACCES: simulated"), {
209+
code: "EACCES",
210+
});
211+
}
212+
return realOpen(...args);
213+
}) as typeof fs.open);
214+
215+
await expect(
216+
withFileLock(lockPath, async () => "ran", fast),
217+
).rejects.toThrow("EACCES: simulated");
156218
});
157219

158220
it("acquires the lock at once when the holder releases it mid-check", async () => {
159221
await fs.writeFile(lockPath, "held");
160-
vi.spyOn(fs, "readFile").mockImplementationOnce((async () => {
161-
await fs.unlink(lockPath);
162-
throw Object.assign(new Error("ENOENT: gone"), { code: "ENOENT" });
163-
}) as typeof fs.readFile);
222+
onceOnOpen(lockPath, "r", () => fs.unlink(lockPath));
164223

165224
expect(await withFileLock(lockPath, async () => "ran", fast)).toBe("ran");
166225
});
@@ -198,6 +257,12 @@ describe("withFileLock", () => {
198257
handle.writeFile = async () => {
199258
throw new Error("EIO: simulated write failure");
200259
};
260+
// The cleanup's own close failing must not mask the write failure.
261+
const realClose = handle.close.bind(handle);
262+
handle.close = async () => {
263+
await realClose();
264+
throw new Error("EIO: simulated close failure");
265+
};
201266
return handle;
202267
});
203268

@@ -214,6 +279,16 @@ describe("withFileLock", () => {
214279
expect(await fs.readFile(lockPath, "utf-8")).toBe("someone else");
215280
});
216281

282+
it("releases quietly when its lock is already gone", async () => {
283+
expect(
284+
await withFileLock(lockPath, async () => {
285+
await fs.unlink(lockPath);
286+
return "ran";
287+
}),
288+
).toBe("ran");
289+
expect(await exists(lockPath)).toBe(false);
290+
});
291+
217292
it("fails when the lock file cannot be created", async () => {
218293
const missing = path.join(dir, "no-such-dir", "memory.jsonl.lock");
219294
await expect(withFileLock(missing, async () => "ran")).rejects.toThrow(

‎src/memory/index.ts‎

Lines changed: 68 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,9 @@ export interface FileLockOptions {
114114
retryMs?: number;
115115
}
116116

117+
// For best-effort lock cleanup, whose failure must not mask the real outcome.
118+
function ignoreError(): void {}
119+
117120
// Whether the process that wrote a lock file is known to be gone. Only a
118121
// process on this host can be checked; for any other host, or a lock whose
119122
// contents cannot be read, the lock's age decides instead.
@@ -137,6 +140,63 @@ function isLockOwnerDead(contents: string): boolean {
137140
}
138141
}
139142

143+
// Whether the lock at lockPath is abandoned. Its owner and its mtime are read
144+
// through one open handle, so both describe the same lock file even if it is
145+
// replaced meanwhile.
146+
async function isLockStale(
147+
lockPath: string,
148+
staleMs: number,
149+
): Promise<boolean> {
150+
const handle = await fs.open(lockPath, "r");
151+
try {
152+
const [contents, stats] = await Promise.all([
153+
handle.readFile("utf-8"),
154+
handle.stat(),
155+
]);
156+
return isLockOwnerDead(contents) || Date.now() - stats.mtimeMs > staleMs;
157+
} finally {
158+
await handle.close();
159+
}
160+
}
161+
162+
// Remove an abandoned lock, returning whether it did. Breaking is itself
163+
// guarded by a second O_EXCL file, so only one waiter breaks at a time and
164+
// re-checks the lock under that guard: two waiters can never both judge the
165+
// same abandoned lock stale and one then remove the lock the other has just
166+
// taken in its place.
167+
async function breakStaleLock(
168+
lockPath: string,
169+
staleMs: number,
170+
): Promise<boolean> {
171+
const breakerPath = `${lockPath}.break`;
172+
let breaker: FileHandle;
173+
try {
174+
breaker = await fs.open(breakerPath, "wx");
175+
} catch (error) {
176+
if ((error as NodeJS.ErrnoException).code !== "EEXIST") {
177+
throw error;
178+
}
179+
// Another waiter is breaking the lock. Its guard is held for a moment
180+
// only, so one that outlives staleMs was left by a crash: clear it.
181+
const stats = await fs.stat(breakerPath).catch(() => null);
182+
if (stats && Date.now() - stats.mtimeMs > staleMs) {
183+
await fs.unlink(breakerPath).catch(ignoreError);
184+
}
185+
return false;
186+
}
187+
try {
188+
await breaker.close();
189+
if (!(await isLockStale(lockPath, staleMs))) {
190+
return false;
191+
}
192+
console.error(`Breaking stale memory file lock ${lockPath}`);
193+
await fs.unlink(lockPath);
194+
return true;
195+
} finally {
196+
await fs.unlink(breakerPath).catch(ignoreError);
197+
}
198+
}
199+
140200
// Run operation while holding an exclusive lock file at lockPath, so that
141201
// mutations from separate server processes sharing one graph file take turns
142202
// and each one loads the graph the previous one saved (#4797). The lock is a
@@ -172,8 +232,8 @@ export async function withFileLock<T>(
172232
} catch (error) {
173233
// The lock was created but not written: remove it rather than leave
174234
// every other writer waiting for it to go stale.
175-
await handle.close().catch(() => {});
176-
await fs.unlink(lockPath).catch(() => {});
235+
await handle.close().catch(ignoreError);
236+
await fs.unlink(lockPath).catch(ignoreError);
177237
throw error;
178238
}
179239
break;
@@ -182,17 +242,10 @@ export async function withFileLock<T>(
182242
// Someone else holds the lock. Break it if it is abandoned; otherwise
183243
// wait and try again.
184244
try {
185-
const [contents, stats] = await Promise.all([
186-
fs.readFile(lockPath, "utf-8"),
187-
fs.stat(lockPath),
188-
]);
189-
if (isLockOwnerDead(contents) || Date.now() - stats.mtimeMs > staleMs) {
190-
// Re-read just before removing, so a lock another waiter has just
191-
// re-acquired after breaking the same stale lock is left alone.
192-
if ((await fs.readFile(lockPath, "utf-8")) === contents) {
193-
console.error(`Breaking stale memory file lock ${lockPath}`);
194-
await fs.unlink(lockPath);
195-
}
245+
if (
246+
(await isLockStale(lockPath, staleMs)) &&
247+
(await breakStaleLock(lockPath, staleMs))
248+
) {
196249
continue;
197250
}
198251
} catch (error) {
@@ -217,7 +270,7 @@ export async function withFileLock<T>(
217270
// staleMs (a large graph, a slow disk) is not mistaken for an abandoned one.
218271
const heartbeat = setInterval(() => {
219272
const now = new Date();
220-
fs.utimes(lockPath, now, now).catch(() => {});
273+
fs.utimes(lockPath, now, now).catch(ignoreError);
221274
}, staleMs / 2);
222275
heartbeat.unref();
223276

@@ -229,7 +282,7 @@ export async function withFileLock<T>(
229282
// the file may now belong to another process.
230283
const current = await fs.readFile(lockPath, "utf-8").catch(() => null);
231284
if (current === token) {
232-
await fs.unlink(lockPath).catch(() => {});
285+
await fs.unlink(lockPath).catch(ignoreError);
233286
}
234287
}
235288
}

0 commit comments

Comments
 (0)