Skip to content

Commit 9af92d5

Browse files
committed
fix: keep null-start-time lock owners fail-closed
Review P1 on #1672: the started-after-acquisition bound compared a wall-clock birth estimate (Date.now() - ps etime) against the persisted wall-clock acquiredAtMs, so a clock step larger than the fixed 30s slack could condemn a live null-start owner and let a waiter steal the held lock — the failure class this PR eliminates. There is no same-clock-domain proof of birth order to be had here (macOS ps etime is itself wall-derived), so drop the bound: an alive pid with no recorded start time is never reclaimed. Regression covers the steal at both the classifier and the lock level.
1 parent 126e309 commit 9af92d5

6 files changed

Lines changed: 40 additions & 126 deletions

File tree

‎src/utils/__tests__/host-process-liveness.test.ts‎

Lines changed: 4 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ vi.mock('../exec.ts', async () => {
88
return { ...actual, runCmdSync: mockRunCmdSync };
99
});
1010

11-
import { isProcessZombie, readProcessStartedAtMs } from '../host-process.ts';
11+
import { isProcessZombie } from '../host-process.ts';
1212

1313
function psReturns(stdout: string, exitCode = 0): void {
1414
mockRunCmdSync.mockReturnValue({ stdout, stderr: '', exitCode });
@@ -34,25 +34,7 @@ test('isProcessZombie treats running states and ps failures as not zombie', () =
3434
assert.equal(isProcessZombie(4242), false);
3535
});
3636

37-
test('readProcessStartedAtMs derives the start from mm:ss elapsed time', () => {
38-
psReturns(' 04:05\n');
39-
assert.equal(readProcessStartedAtMs(4242, 1_000_000), 1_000_000 - (4 * 60 + 5) * 1_000);
40-
});
41-
42-
test('readProcessStartedAtMs handles hh:mm:ss and dd-hh:mm:ss elapsed formats', () => {
43-
psReturns('1:02:03\n');
44-
assert.equal(readProcessStartedAtMs(4242, 10_000_000), 10_000_000 - 3_723_000);
45-
psReturns('18-13:56:27\n');
46-
assert.equal(
47-
readProcessStartedAtMs(4242, 2_000_000_000),
48-
2_000_000_000 - (((18 * 24 + 13) * 60 + 56) * 60 + 27) * 1_000,
49-
);
50-
});
51-
52-
test('readProcessStartedAtMs returns null for unparseable or failed ps output', () => {
53-
psReturns('garbage\n');
54-
assert.equal(readProcessStartedAtMs(4242, 1_000_000), null);
55-
psReturns('', 1);
56-
assert.equal(readProcessStartedAtMs(4242, 1_000_000), null);
57-
assert.equal(readProcessStartedAtMs(-1, 1_000_000), null);
37+
test('isProcessZombie returns false for an invalid pid without invoking ps', () => {
38+
assert.equal(isProcessZombie(-1), false);
39+
assert.equal(mockRunCmdSync.mock.calls.length, 0);
5840
});
Lines changed: 5 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,16 @@
11
import assert from 'node:assert/strict';
22
import { beforeEach, test, vi } from 'vitest';
33

4-
const {
5-
mockIsProcessAlive,
6-
mockIsProcessZombie,
7-
mockReadProcessStartTime,
8-
mockReadProcessStartedAtMs,
9-
} = vi.hoisted(() => ({
4+
const { mockIsProcessAlive, mockIsProcessZombie, mockReadProcessStartTime } = vi.hoisted(() => ({
105
mockIsProcessAlive: vi.fn(),
116
mockIsProcessZombie: vi.fn(),
127
mockReadProcessStartTime: vi.fn(),
13-
mockReadProcessStartedAtMs: vi.fn(),
148
}));
159

1610
vi.mock('../host-process.ts', () => ({
1711
isProcessAlive: mockIsProcessAlive,
1812
isProcessZombie: mockIsProcessZombie,
1913
readProcessStartTime: mockReadProcessStartTime,
20-
readProcessStartedAtMs: mockReadProcessStartedAtMs,
2114
}));
2215

2316
import { classifyOwnerLiveness } from '../owner-identity.ts';
@@ -28,7 +21,6 @@ beforeEach(() => {
2821
mockIsProcessAlive.mockReset().mockReturnValue(true);
2922
mockIsProcessZombie.mockReset().mockReturnValue(false);
3023
mockReadProcessStartTime.mockReset().mockReturnValue('start-a');
31-
mockReadProcessStartedAtMs.mockReset().mockReturnValue(null);
3224
});
3325

3426
test('classifies a zombie owner as owner-process-dead despite a matching start time', () => {
@@ -52,35 +44,9 @@ test('a definite start-time mismatch classifies as owner-process-dead', () => {
5244
);
5345
});
5446

55-
test('null recorded start time stays live without an acquisition bound', () => {
56-
mockReadProcessStartedAtMs.mockReturnValue(Date.now());
47+
test('a null-start-time owner stays fail-closed while its pid is alive', () => {
48+
// No same-clock-domain proof of birth order exists for a null-start owner:
49+
// a clock step could make a live owner look like it started after the
50+
// resource was acquired, so an alive pid must never be condemned on age.
5751
assert.equal(classifyOwnerLiveness({ owner: { pid: OWNER_PID, startTime: null } }), 'live');
5852
});
59-
60-
test('null recorded start time dies when the pid started after acquisition', () => {
61-
const acquiredAtMs = 1_000_000;
62-
mockReadProcessStartedAtMs.mockReturnValue(acquiredAtMs + 120_000);
63-
assert.equal(
64-
classifyOwnerLiveness({ owner: { pid: OWNER_PID, startTime: null }, acquiredAtMs }),
65-
'owner-process-dead',
66-
);
67-
});
68-
69-
test('null recorded start time stays live when the pid started before acquisition or within slack', () => {
70-
const acquiredAtMs = 1_000_000;
71-
mockReadProcessStartedAtMs.mockReturnValue(acquiredAtMs - 5_000);
72-
assert.equal(
73-
classifyOwnerLiveness({ owner: { pid: OWNER_PID, startTime: null }, acquiredAtMs }),
74-
'live',
75-
);
76-
mockReadProcessStartedAtMs.mockReturnValue(acquiredAtMs + 5_000);
77-
assert.equal(
78-
classifyOwnerLiveness({ owner: { pid: OWNER_PID, startTime: null }, acquiredAtMs }),
79-
'live',
80-
);
81-
mockReadProcessStartedAtMs.mockReturnValue(null);
82-
assert.equal(
83-
classifyOwnerLiveness({ owner: { pid: OWNER_PID, startTime: null }, acquiredAtMs }),
84-
'live',
85-
);
86-
});

‎src/utils/__tests__/process-lock.test.ts‎

Lines changed: 21 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ vi.mock('../host-process.ts', async (importOriginal) => {
1212
});
1313

1414
import { acquireProcessLock, type ProcessLockOwner } from '../process-lock.ts';
15-
import { readProcessStartedAtMs, readProcessStartTime } from '../host-process.ts';
15+
import { readProcessStartTime } from '../host-process.ts';
1616
import { mkdtempForTestSync } from '../../__tests__/test-utils/tmp-dir.ts';
1717

1818
let tmpDir: string;
@@ -84,34 +84,35 @@ test('acquireProcessLock reclaims locks owned by zombie processes', async () =>
8484
}
8585
});
8686

87-
test('acquireProcessLock reclaims a null-start-time lock whose pid started after acquisition', async () => {
88-
const ownStartedAtMs = readProcessStartedAtMs(process.pid);
89-
if (ownStartedAtMs === null) {
90-
// ps lost to CPU contention; the classification itself is covered by
91-
// owner-identity-liveness.test.ts.
92-
return;
93-
}
94-
const lockDirPath = path.join(tmpDir, 'recycled.lock');
87+
test('acquireProcessLock never steals a null-start-time lock from an alive pid', async () => {
88+
const lockDirPath = path.join(tmpDir, 'null-start.lock');
9589
fs.mkdirSync(lockDirPath);
96-
// Recorded before this process started, with no start-time identity: the
97-
// pid provably belongs to a later process, so the lock is reclaimable.
90+
// An acquiredAtMs far older than this process simulates what a wall-clock
91+
// step makes a live null-start owner look like; age is not proof of death,
92+
// so the waiter must time out instead of reclaiming the held lock.
9893
fs.writeFileSync(
9994
path.join(lockDirPath, 'owner.json'),
10095
JSON.stringify({
10196
pid: process.pid,
10297
startTime: null,
103-
acquiredAtMs: ownStartedAtMs - 10 * 60_000,
98+
acquiredAtMs: Date.now() - 365 * 24 * 60 * 60_000,
10499
}),
105100
);
106101

107-
const release = await acquireProcessLock({
108-
lockDirPath,
109-
owner: currentProcessOwner(),
110-
timeoutMs: 3_000,
111-
pollMs: 1,
112-
});
113-
await release();
114-
assert.equal(fs.existsSync(lockDirPath), false);
102+
await assert.rejects(
103+
() =>
104+
acquireProcessLock({
105+
lockDirPath,
106+
owner: currentProcessOwner(),
107+
timeoutMs: 50,
108+
pollMs: 1,
109+
}),
110+
(error: unknown) => {
111+
assert.ok(error instanceof AppError);
112+
assert.equal(error.details?.ownerLiveness, 'live');
113+
return true;
114+
},
115+
);
115116
});
116117

117118
test('acquireProcessLock reports live lock owner details on timeout', async () => {

‎src/utils/host-process.ts‎

Lines changed: 1 addition & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -62,25 +62,7 @@ export function isProcessZombie(pid: number): boolean {
6262
return readProcessField(pid, 'state=')?.startsWith('Z') ?? false;
6363
}
6464

65-
export function readProcessStartedAtMs(pid: number, nowMs: number = Date.now()): number | null {
66-
const elapsedMs = parseEtimeMs(readProcessField(pid, 'etime='));
67-
return elapsedMs === null ? null : nowMs - elapsedMs;
68-
}
69-
70-
function parseEtimeMs(value: string | null): number | null {
71-
if (!value) return null;
72-
const match = /^(?:(\d+)-)?(?:(\d+):)?(\d{1,2}):(\d{2})$/.exec(value);
73-
if (!match) return null;
74-
const [, days = '0', hours = '0', minutes, seconds] = match;
75-
return (
76-
(Number(days) * 86_400 + Number(hours) * 3_600 + Number(minutes) * 60 + Number(seconds)) * 1_000
77-
);
78-
}
79-
80-
function readProcessField(
81-
pid: number,
82-
field: 'lstart=' | 'command=' | 'state=' | 'etime=',
83-
): string | null {
65+
function readProcessField(pid: number, field: 'lstart=' | 'command=' | 'state='): string | null {
8466
if (!Number.isInteger(pid) || pid <= 0) return null;
8567
try {
8668
const result = runCmdSync('ps', ['-p', String(pid), '-o', field], {

‎src/utils/owner-identity.ts‎

Lines changed: 7 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,5 @@
11
import fs from 'node:fs';
2-
import {
3-
isProcessAlive,
4-
isProcessZombie,
5-
readProcessStartedAtMs,
6-
readProcessStartTime,
7-
} from './host-process.ts';
8-
9-
// ps reports process start times at second granularity and the host clock can
10-
// step; a current start time must clear acquiredAtMs by this margin before it
11-
// proves the pid was recycled.
12-
const OWNER_START_TIME_SLACK_MS = 30_000;
2+
import { isProcessAlive, isProcessZombie, readProcessStartTime } from './host-process.ts';
133

144
export type OwnerIdentity = {
155
pid: number;
@@ -37,29 +27,24 @@ export function ownerIdentityMatches(
3727
* loses under CPU contention) is not proof the owner died, so it never
3828
* condemns a pid that kill(pid, 0) says is alive. Death is only concluded
3929
* from positive evidence: the pid is gone, the process is a zombie (already
40-
* terminated, merely unreaped), its start time differs from the recorded one,
41-
* or it started after `acquiredAtMs` — a process born after the resource was
42-
* acquired cannot be the acquirer, which catches recycled pids even when the
43-
* owner's start time was never recorded.
30+
* terminated, merely unreaped), or its start time differs from the recorded
31+
* one. An owner recorded without a start time stays fail-closed while its pid
32+
* is alive: there is no same-clock-domain proof of birth order (wall-clock
33+
* arithmetic over `ps etime` shifts under clock steps), and misreading a live
34+
* owner as recycled would let a waiter steal a held resource.
4435
*/
4536
export function classifyOwnerLiveness(params: {
4637
owner: Pick<OwnerIdentity, 'pid' | 'startTime'>;
4738
stateDir?: string;
48-
acquiredAtMs?: number;
4939
}): OwnerLiveness {
50-
const { owner, stateDir, acquiredAtMs } = params;
40+
const { owner, stateDir } = params;
5141
if (!isProcessAlive(owner.pid)) return 'owner-process-dead';
5242
if (isProcessZombie(owner.pid)) return 'owner-process-dead';
5343
if (owner.startTime) {
5444
const currentStartTime = readProcessStartTime(owner.pid);
5545
if (currentStartTime !== null && currentStartTime !== owner.startTime) {
5646
return 'owner-process-dead';
5747
}
58-
} else if (acquiredAtMs !== undefined) {
59-
const startedAtMs = readProcessStartedAtMs(owner.pid);
60-
if (startedAtMs !== null && startedAtMs > acquiredAtMs + OWNER_START_TIME_SLACK_MS) {
61-
return 'owner-process-dead';
62-
}
6348
}
6449
if (!stateDir) return 'live';
6550
try {

‎src/utils/process-lock.ts‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -117,14 +117,12 @@ function readProcessLockDiagnostics(
117117
ownerPid: owner.pid,
118118
ownerStartTime: owner.startTime,
119119
ownerAgeMs: Math.max(0, Math.round(nowMs - owner.acquiredAtMs)),
120-
ownerLiveness: classifyOwnerLiveness({ owner, acquiredAtMs: owner.acquiredAtMs }),
120+
ownerLiveness: classifyOwnerLiveness({ owner }),
121121
}
122122
: {}),
123123
};
124124
}
125125

126126
function isLiveProcessLockOwner(owner: ProcessLockOwner): boolean {
127-
return (
128-
classifyOwnerLiveness({ owner, acquiredAtMs: owner.acquiredAtMs }) !== 'owner-process-dead'
129-
);
127+
return classifyOwnerLiveness({ owner }) !== 'owner-process-dead';
130128
}

0 commit comments

Comments
 (0)