Skip to content

Commit bcc54ec

Browse files
fix(memory): make stale-lock reclamation ownership-safe
Review found two races in the first cut: N waiters that all saw a stale lock could each unlink it, so a slow waiter deleted the fast waiter's fresh lock and both proceeded; and an operation longer than the stale threshold had its live lock stolen, after which its finally removed the successor's lock. Both reintroduce the lost update this PR exists to prevent. - the lock file now holds the holder's token - a live holder heartbeats the lock's mtime, so a long operation is never mistaken for a crashed one - stale takeover goes through rename(2): of N reclaimers exactly one wins, the rest see ENOENT and wait on the winner's lock - release unlinks only if the file still holds our token Tests: eight simultaneous reclaimers of one stale lock keep 8/8 entities and leave no lock behind; a slow heartbeating holder is not stolen from and its release does not remove the successor's lock.
1 parent 6393874 commit bcc54ec

2 files changed

Lines changed: 91 additions & 14 deletions

File tree

‎src/memory/__tests__/cross-process-lock.test.ts‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,4 +52,37 @@ describe('KnowledgeGraphManager cross-process exclusion', () => {
5252
await manager.createEntities([entity('x')]);
5353
expect((await manager.readGraph()).entities).toHaveLength(1);
5454
});
55+
56+
it('lets exactly one of many simultaneous stale-lock reclaimers win', async () => {
57+
// The review case: N waiters all observe the same stale lock. A bare
58+
// unlink+open lets the slow waiter delete the fast waiter's fresh lock.
59+
await fs.mkdir(testDir, { recursive: true });
60+
await fs.writeFile(`${testFilePath}.lock`, 'dead-holder');
61+
const old = new Date(Date.now() - 60_000);
62+
await fs.utimes(`${testFilePath}.lock`, old, old);
63+
64+
const managers = Array.from({ length: 8 }, () => new KnowledgeGraphManager(testFilePath));
65+
await Promise.all(managers.map((m, i) => m.createEntities([entity(`r${i}`)])));
66+
67+
const names = (await managers[0].readGraph()).entities.map(e => e.name).sort();
68+
expect(names).toEqual(Array.from({ length: 8 }, (_, i) => `r${i}`).sort());
69+
await expect(fs.stat(`${testFilePath}.lock`)).rejects.toMatchObject({ code: 'ENOENT' });
70+
});
71+
72+
it("does not let a slow holder's release remove a successor's lock", async () => {
73+
// A heartbeating holder must not be stolen from, and a release must never
74+
// unlink a lock it does not own.
75+
const a = new KnowledgeGraphManager(testFilePath);
76+
const b = new KnowledgeGraphManager(testFilePath);
77+
const slow = (a as any).withFileLock(async () => {
78+
await new Promise(r => setTimeout(r, 400));
79+
return 'a-done';
80+
});
81+
await new Promise(r => setTimeout(r, 50));
82+
const fast = b.createEntities([entity('b1')]);
83+
expect(await slow).toBe('a-done');
84+
await fast;
85+
expect((await a.readGraph()).entities.map(e => e.name)).toEqual(['b1']);
86+
await expect(fs.stat(`${testFilePath}.lock`)).rejects.toMatchObject({ code: 'ENOENT' });
87+
});
5588
});

‎src/memory/index.ts‎

Lines changed: 58 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -113,10 +113,20 @@ export class KnowledgeGraphManager {
113113
// each run their own queue and still overwrite each other's load→mutate→save
114114
// (#1819, #3286). Measured on this commit: 20 writes split across two
115115
// processes keep exactly 10 — each process serialises its own half and the
116-
// last one to write the file wins. A sidecar lock file created with O_EXCL is
117-
// atomic on POSIX and Windows and needs no dependency. A lock older than
118-
// LOCK_STALE_MS is treated as left behind by a crashed process.
116+
// last one to write the file wins.
117+
//
118+
// The lock is a sidecar file created with O_EXCL (atomic on POSIX and
119+
// Windows, no dependency) that contains the holder's token. Three rules make
120+
// it ownership-safe, which a bare unlink-and-retry is not:
121+
// 1. A live holder heartbeats the lock's mtime, so a long operation is
122+
// never mistaken for a crashed one.
123+
// 2. Stale takeover goes through rename(2), not unlink: of N waiters that
124+
// all saw the same stale lock, exactly one rename succeeds; the others
125+
// get ENOENT and retry against the new holder's lock.
126+
// 3. Release removes the lock only if it still holds our token, so a
127+
// holder can never delete a replacement's lock.
119128
private static readonly LOCK_STALE_MS = 30_000;
129+
private static readonly LOCK_HEARTBEAT_MS = 5_000;
120130
private static readonly LOCK_RETRY_MS = 15;
121131
private static readonly LOCK_TIMEOUT_MS = 10_000;
122132

@@ -126,37 +136,71 @@ export class KnowledgeGraphManager {
126136

127137
private async withFileLock<T>(operation: () => Promise<T>): Promise<T> {
128138
await fs.mkdir(path.dirname(this.memoryFilePath), { recursive: true }).catch(() => {});
139+
const token = `${process.pid}:${randomBytes(12).toString('hex')}`;
129140
const deadline = Date.now() + KnowledgeGraphManager.LOCK_TIMEOUT_MS;
130141
for (;;) {
131142
let handle: fs.FileHandle;
132143
try {
133144
handle = await fs.open(this.lockFilePath, 'wx');
134145
} catch (error) {
135146
if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error;
136-
try {
137-
const stat = await fs.stat(this.lockFilePath);
138-
if (Date.now() - stat.mtimeMs > KnowledgeGraphManager.LOCK_STALE_MS) {
139-
await fs.unlink(this.lockFilePath).catch(() => {});
140-
continue;
141-
}
142-
} catch {
143-
continue; // the holder released between open and stat; retry now
144-
}
147+
await this.reclaimIfStale();
145148
if (Date.now() > deadline) {
146149
throw new Error(`Timed out waiting for lock ${this.lockFilePath}`);
147150
}
148151
await new Promise(resolve => setTimeout(resolve, KnowledgeGraphManager.LOCK_RETRY_MS));
149152
continue;
150153
}
154+
// Acquired. Record who holds it, then keep it visibly alive.
155+
await handle.writeFile(token);
156+
await handle.close().catch(() => {});
157+
const heartbeat = setInterval(() => {
158+
const now = new Date();
159+
fs.utimes(this.lockFilePath, now, now).catch(() => {});
160+
}, KnowledgeGraphManager.LOCK_HEARTBEAT_MS);
151161
try {
152162
return await operation();
153163
} finally {
154-
await handle.close().catch(() => {});
155-
await fs.unlink(this.lockFilePath).catch(() => {});
164+
clearInterval(heartbeat);
165+
await this.releaseIfOwned(token);
156166
}
157167
}
158168
}
159169

170+
// Take over a lock whose holder has stopped heartbeating. rename(2) is the
171+
// arbiter: N waiters may all decide the lock is stale, but only the first
172+
// rename succeeds; the rest fail with ENOENT and go back to waiting on
173+
// whatever lock the winner creates next.
174+
private async reclaimIfStale(): Promise<void> {
175+
let stat;
176+
try {
177+
stat = await fs.stat(this.lockFilePath);
178+
} catch {
179+
return; // released between our open() and stat(); retry immediately
180+
}
181+
if (Date.now() - stat.mtimeMs <= KnowledgeGraphManager.LOCK_STALE_MS) return;
182+
const tomb = `${this.lockFilePath}.stale.${randomBytes(8).toString('hex')}`;
183+
try {
184+
await fs.rename(this.lockFilePath, tomb);
185+
} catch {
186+
return; // another waiter won the reclaim; fall through to retry
187+
}
188+
await fs.unlink(tomb).catch(() => {});
189+
}
190+
191+
// Remove the lock only if it is still ours. If a (wrongly) stale-reclaimed
192+
// lock has since been taken by someone else, their token is in the file and
193+
// we must leave it alone.
194+
private async releaseIfOwned(token: string): Promise<void> {
195+
try {
196+
const current = await fs.readFile(this.lockFilePath, 'utf8');
197+
if (current !== token) return;
198+
await fs.unlink(this.lockFilePath);
199+
} catch {
200+
// already gone, or unreadable: nothing of ours to remove
201+
}
202+
}
203+
160204
private async loadGraph(): Promise<KnowledgeGraph> {
161205
try {
162206
const data = await fs.readFile(this.memoryFilePath, "utf-8");

0 commit comments

Comments
 (0)