Skip to content

Commit 2ec6027

Browse files
committed
fix(host-kit): process lock proves ownership before release and never evicts a live owner
Release removed the lock directory unconditionally, so a contender that reclaimed the lock while its previous holder was away had that holder delete the exclusion belonging to whoever holds it now. Release removes only while the record inside still names the acquirer, and says so with a typed reason when it cannot read far enough to tell, rather than clearing a lock it might not own. A record that could not be read looked exactly like a record that was never written, so a live owner whose owner.json is unreadable or malformed aged out at the grace window and lost a lock it was holding. Only ENOENT now speaks for an unwritten record; any other read failure, and any record that does not name a live-shaped process, is an owner of unknown identity and the lock stands. Both wait paths carry a hint naming the directory, because nothing else in the repository removes it. A stale reclaim moves the abandoned directory aside under a unique `.lock` name before removing it. Two plain removals both report success, because the second is a silent no-op on a path the first already cleared, and both contenders then continue as though they had freed the lock. mkdir's EEXIST, unchanged, still arbitrates the lock itself. A reclaim that fails for a reason the fallback cannot settle waits and retries instead of surfacing an errno, and where the win32 fallback does force a removal it re-reads the record first, because a refused rename means a live contender may have claimed the path since. The allocation store keeps lane locks beside the lanes they guard, so it now recognises any `.lock` directory as a lock instead of only the `.lane.lock` suffix.
1 parent 28accb8 commit 2ec6027

4 files changed

Lines changed: 491 additions & 24 deletions

File tree

‎packages/host-kit/src/internal/process-lock.test.ts‎

Lines changed: 312 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ import { acquireProcessLock, type ProcessLockOwner } from './process-lock.ts';
1515
import { readProcessStartTime } from './host-process.ts';
1616
import { mkdtempForTestSync } from './tmp-dir.fixtures.ts';
1717

18+
const RECLAIMED_MARK = '.reclaimed-';
19+
1820
let tmpDir: string;
1921

2022
beforeEach(() => {
@@ -142,6 +144,316 @@ test('acquireProcessLock reports live lock owner details on timeout', async () =
142144
);
143145
});
144146

147+
test('release leaves a lock whose record names a different process', async () => {
148+
const lockDirPath = path.join(tmpDir, 'taken-over.lock');
149+
const ownerFilePath = path.join(lockDirPath, 'owner.json');
150+
const release = await acquireProcessLock({
151+
lockDirPath,
152+
owner: currentProcessOwner(),
153+
});
154+
155+
// A contender that reclaimed this lock while we were away republished the record
156+
// with its own identity; removing the directory would give away its lock.
157+
fs.writeFileSync(
158+
ownerFilePath,
159+
JSON.stringify({ pid: 999_999_999, startTime: null, acquiredAtMs: Date.now() }),
160+
);
161+
await release();
162+
assert.equal(fs.existsSync(ownerFilePath), true);
163+
});
164+
165+
test('acquireProcessLock does not evict a live owner whose owner.json is malformed', async () => {
166+
const lockDirPath = path.join(tmpDir, 'malformed.lock');
167+
fs.mkdirSync(lockDirPath);
168+
fs.writeFileSync(path.join(lockDirPath, 'owner.json'), '{ pid: ');
169+
stampDirectoryAbandoned(lockDirPath);
170+
171+
await assert.rejects(
172+
() =>
173+
acquireProcessLock({
174+
lockDirPath,
175+
owner: currentProcessOwner(),
176+
timeoutMs: 50,
177+
pollMs: 1,
178+
}),
179+
(error: unknown) => {
180+
assert.ok(error instanceof AppError);
181+
assert.equal(error.details?.ownerRecordUnreadable, true);
182+
return true;
183+
},
184+
);
185+
assert.equal(fs.existsSync(lockDirPath), true);
186+
});
187+
188+
test('acquireProcessLock does not evict an owner record it cannot read', async () => {
189+
const lockDirPath = path.join(tmpDir, 'unreadable.lock');
190+
fs.mkdirSync(lockDirPath);
191+
// A directory where the record belongs fails the read with EISDIR rather than
192+
// ENOENT, which is a live owner we know nothing about, not an unwritten one.
193+
fs.mkdirSync(path.join(lockDirPath, 'owner.json'));
194+
stampDirectoryAbandoned(lockDirPath);
195+
196+
await assert.rejects(
197+
() =>
198+
acquireProcessLock({
199+
lockDirPath,
200+
owner: currentProcessOwner(),
201+
timeoutMs: 50,
202+
pollMs: 1,
203+
}),
204+
(error: unknown) => {
205+
assert.ok(error instanceof AppError);
206+
assert.equal(error.details?.ownerRecordUnreadable, true);
207+
return true;
208+
},
209+
);
210+
assert.equal(fs.existsSync(lockDirPath), true);
211+
});
212+
213+
test('acquireProcessLock reclaims a lock whose owner record was never written', async () => {
214+
const lockDirPath = path.join(tmpDir, 'unpublished.lock');
215+
fs.mkdirSync(lockDirPath);
216+
stampDirectoryAbandoned(lockDirPath);
217+
218+
const release = await acquireProcessLock({
219+
lockDirPath,
220+
owner: currentProcessOwner(),
221+
timeoutMs: 500,
222+
pollMs: 1,
223+
});
224+
await release();
225+
assert.equal(fs.existsSync(lockDirPath), false);
226+
});
227+
228+
test('one abandoned lock offered to two contenders is held by exactly one of them', async () => {
229+
const lockDirPath = path.join(tmpDir, 'contended.lock');
230+
fs.mkdirSync(lockDirPath);
231+
stampDirectoryAbandoned(lockDirPath);
232+
233+
const attempts = await Promise.allSettled([
234+
acquireProcessLock({
235+
lockDirPath,
236+
owner: currentProcessOwner(),
237+
ownerGraceMs: 0,
238+
timeoutMs: 250,
239+
pollMs: 2,
240+
}),
241+
acquireProcessLock({
242+
lockDirPath,
243+
owner: currentProcessOwner(),
244+
ownerGraceMs: 0,
245+
timeoutMs: 250,
246+
pollMs: 2,
247+
}),
248+
]);
249+
const acquired = attempts.filter((attempt) => attempt.status === 'fulfilled');
250+
const refused = attempts.filter((attempt) => attempt.status === 'rejected');
251+
252+
assert.equal(acquired.length, 1);
253+
assert.equal(refused.length, 1);
254+
const reason = (refused[0] as PromiseRejectedResult).reason;
255+
assert.ok(reason instanceof AppError);
256+
assert.equal(reason.details?.ownerLiveness, 'live');
257+
await (acquired[0] as PromiseFulfilledResult<() => Promise<void>>).value();
258+
assert.deepEqual(listReclaimedSiblings(tmpDir), []);
259+
});
260+
261+
function listReclaimedSiblings(directory: string): string[] {
262+
return fs
263+
.readdirSync(directory)
264+
.filter((entry) => entry.includes(RECLAIMED_MARK))
265+
.sort();
266+
}
267+
268+
const UNINFORMATIVE_OWNER_RECORDS = [
269+
'null',
270+
'"999999999"',
271+
'{"pid":"999999999","startTime":null,"acquiredAtMs":1}',
272+
'{"pid":0,"startTime":null,"acquiredAtMs":1}',
273+
'{"pid":999999999,"startTime":7,"acquiredAtMs":1}',
274+
'{"pid":999999999,"startTime":null}',
275+
] as const;
276+
277+
for (const [index, record] of UNINFORMATIVE_OWNER_RECORDS.entries()) {
278+
test(`acquireProcessLock does not evict the lock behind the record ${record}`, async () => {
279+
const lockDirPath = path.join(tmpDir, `uninformative-${index}.lock`);
280+
fs.mkdirSync(lockDirPath);
281+
fs.writeFileSync(path.join(lockDirPath, 'owner.json'), record);
282+
stampDirectoryAbandoned(lockDirPath);
283+
284+
await assert.rejects(
285+
() =>
286+
acquireProcessLock({
287+
lockDirPath,
288+
owner: currentProcessOwner(),
289+
timeoutMs: 50,
290+
pollMs: 1,
291+
}),
292+
(error: unknown) => {
293+
assert.ok(error instanceof AppError);
294+
assert.equal(error.details?.ownerRecordUnreadable, true);
295+
return true;
296+
},
297+
);
298+
assert.equal(fs.existsSync(lockDirPath), true);
299+
});
300+
}
301+
302+
test('release reports a lock whose owner record it cannot read instead of clearing it', async () => {
303+
const lockDirPath = path.join(tmpDir, 'unverifiable-release.lock');
304+
const release = await acquireProcessLock({
305+
lockDirPath,
306+
owner: currentProcessOwner(),
307+
});
308+
fs.rmSync(path.join(lockDirPath, 'owner.json'));
309+
fs.mkdirSync(path.join(lockDirPath, 'owner.json'));
310+
311+
await assert.rejects(
312+
() => release(),
313+
(error: unknown) => {
314+
assert.ok(error instanceof AppError);
315+
assert.equal(error.details?.ownerReleaseUnverified, true);
316+
assert.match(String(error.details?.hint), /unverifiable-release\.lock/);
317+
return true;
318+
},
319+
);
320+
assert.equal(fs.existsSync(lockDirPath), true);
321+
});
322+
323+
test('acquireProcessLock reclaims a stray path in place of the lock directory', async () => {
324+
const lockDirPath = path.join(tmpDir, 'stray.lock');
325+
fs.writeFileSync(lockDirPath, 'not a lock');
326+
const stale = new Date(Date.now() - 60_000);
327+
fs.utimesSync(lockDirPath, stale, stale);
328+
329+
const release = await acquireProcessLock({
330+
lockDirPath,
331+
owner: currentProcessOwner(),
332+
timeoutMs: 500,
333+
pollMs: 1,
334+
});
335+
await release();
336+
assert.equal(fs.existsSync(lockDirPath), false);
337+
});
338+
339+
test('a forced reclaim leaves a directory that a live owner republished', async () => {
340+
const lockDirPath = path.join(tmpDir, 'republished.lock');
341+
const ownerFilePath = path.join(lockDirPath, 'owner.json');
342+
fs.mkdirSync(lockDirPath);
343+
fs.writeFileSync(
344+
ownerFilePath,
345+
JSON.stringify({ pid: 999_999_999, startTime: null, acquiredAtMs: Date.now() }),
346+
);
347+
348+
// A win32 handle refuses the rename, and by the time the forced removal would run a
349+
// live holder has claimed the path: the forced removal must not reach its directory.
350+
const realRename = fs.renameSync;
351+
const renameSpy = vi.spyOn(fs, 'renameSync').mockImplementation(((
352+
from: fs.PathLike,
353+
to: fs.PathLike,
354+
) => {
355+
if (!String(to).includes(RECLAIMED_MARK)) return realRename(from, to);
356+
fs.rmSync(String(from), { recursive: true, force: true });
357+
fs.mkdirSync(String(from));
358+
fs.writeFileSync(path.join(String(from), 'owner.json'), JSON.stringify(currentProcessOwner()));
359+
throw Object.assign(new Error('operation not permitted'), { code: 'EPERM' });
360+
}) as typeof fs.renameSync);
361+
362+
try {
363+
await assert.rejects(
364+
() =>
365+
acquireProcessLock({
366+
lockDirPath,
367+
owner: { pid: 999_999_998, startTime: null, acquiredAtMs: Date.now() },
368+
timeoutMs: 50,
369+
pollMs: 1,
370+
}),
371+
(error: unknown) => {
372+
assert.ok(error instanceof AppError);
373+
assert.equal(error.details?.ownerLiveness, 'live');
374+
return true;
375+
},
376+
);
377+
assert.equal(fs.existsSync(ownerFilePath), true);
378+
} finally {
379+
renameSpy.mockRestore();
380+
}
381+
});
382+
383+
test('a reclaim that cannot remove the directory it moved aside still holds the lock', async () => {
384+
const lockDirPath = path.join(tmpDir, 'immovable.lock');
385+
fs.mkdirSync(lockDirPath);
386+
fs.writeFileSync(
387+
path.join(lockDirPath, 'owner.json'),
388+
JSON.stringify({ pid: 999_999_999, startTime: null, acquiredAtMs: Date.now() }),
389+
);
390+
stampDirectoryAbandoned(lockDirPath);
391+
392+
const realRemove = fs.rmSync;
393+
const removeSpy = vi.spyOn(fs, 'rmSync').mockImplementation(((target: fs.PathLike, options) => {
394+
if (String(target).includes(RECLAIMED_MARK)) {
395+
throw Object.assign(new Error('directory is busy'), { code: 'EBUSY' });
396+
}
397+
return realRemove(target, options);
398+
}) as typeof fs.rmSync);
399+
400+
try {
401+
const release = await acquireProcessLock({
402+
lockDirPath,
403+
owner: currentProcessOwner(),
404+
timeoutMs: 500,
405+
pollMs: 1,
406+
});
407+
assert.equal(fs.existsSync(path.join(lockDirPath, 'owner.json')), true);
408+
await release();
409+
} finally {
410+
removeSpy.mockRestore();
411+
}
412+
});
413+
414+
test('a reclaim whose directory another contender moved aside retries and acquires', async () => {
415+
const lockDirPath = path.join(tmpDir, 'lost-race.lock');
416+
fs.mkdirSync(lockDirPath);
417+
fs.writeFileSync(
418+
path.join(lockDirPath, 'owner.json'),
419+
JSON.stringify({ pid: 999_999_999, startTime: null, acquiredAtMs: Date.now() }),
420+
);
421+
stampDirectoryAbandoned(lockDirPath);
422+
423+
// The contender that loses the rename finds the path already gone and cannot have
424+
// cleared anything; it goes back to `mkdir`, which is what decides the lock.
425+
let attempted = 0;
426+
const realRename = fs.renameSync;
427+
const renameSpy = vi.spyOn(fs, 'renameSync').mockImplementation(((
428+
from: fs.PathLike,
429+
to: fs.PathLike,
430+
) => {
431+
if (!String(to).includes(RECLAIMED_MARK)) return realRename(from, to);
432+
attempted += 1;
433+
fs.rmSync(lockDirPath, { recursive: true, force: true });
434+
throw Object.assign(new Error('no such directory'), { code: 'ENOENT' });
435+
}) as typeof fs.renameSync);
436+
437+
try {
438+
const release = await acquireProcessLock({
439+
lockDirPath,
440+
owner: currentProcessOwner(),
441+
timeoutMs: 500,
442+
pollMs: 1,
443+
});
444+
assert.equal(attempted, 1);
445+
assert.equal(fs.existsSync(path.join(lockDirPath, 'owner.json')), true);
446+
await release();
447+
} finally {
448+
renameSpy.mockRestore();
449+
}
450+
});
451+
452+
function stampDirectoryAbandoned(directory: string): void {
453+
const abandoned = new Date(Date.now() - 60_000);
454+
fs.utimesSync(directory, abandoned, abandoned);
455+
}
456+
145457
function currentProcessOwner(): ProcessLockOwner {
146458
return {
147459
pid: process.pid,

0 commit comments

Comments
 (0)