Skip to content

Commit 4551fb7

Browse files
authored
test: make pid-liveness fixtures deterministic under load (#1556)
Three tests classify a fixture owner's liveness via classifyOwnerLiveness (or the daemon-process equivalent), which re-reads the owner's process start time via a real `ps -p <pid> -o lstart=` shell-out with a 1s timeout. Under full-suite CPU contention that second read can miss its deadline and return null, mismatching the value captured earlier and flipping a genuinely-live owner to 'owner-process-dead'. Pin the pid->start-time (and, for the daemon-client case, pid->command) mapping to a deterministic value per test file instead of letting a second real subprocess call race the first, without weakening the dead-owner path (isProcessAlive stays real and un-mocked everywhere).
1 parent 14d731c commit 4551fb7

3 files changed

Lines changed: 91 additions & 3 deletions

File tree

‎src/__tests__/cli-device-status.test.ts‎

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,38 @@ import assert from 'node:assert/strict';
22
import fs from 'node:fs';
33
import os from 'node:os';
44
import path from 'node:path';
5-
import { test } from 'vitest';
5+
import { test, vi } from 'vitest';
66
import { readCurrentOwnerIdentity } from '../utils/owner-identity.ts';
77
import { runCliCapture } from './cli-capture.ts';
88

9+
// readProcessStartTime shells out to `ps` with a 1s timeout (see
10+
// utils/host-process.ts). This file's fixtures read our own start time once
11+
// to build a "live" claim, then production re-reads it during classification
12+
// to confirm identity; under full-suite CPU contention that second `ps` call
13+
// can miss its deadline and return null, mismatching the cached value and
14+
// flipping a genuinely-live claim to 'owner-process-dead'. Cache our own
15+
// pid's start time after the first (real) read so every later read is
16+
// self-consistent with the fixtures instead of racing a fresh `ps` call.
17+
// isProcessAlive is left real (a plain `kill(pid, 0)` syscall, not a
18+
// subprocess), so the fabricated-dead-pid fixtures below still classify as
19+
// stale through that real, non-flaky check.
20+
vi.mock('../utils/host-process.ts', async () => {
21+
const actual = await vi.importActual<typeof import('../utils/host-process.ts')>(
22+
'../utils/host-process.ts',
23+
);
24+
let cachedOwnStartTime: string | null | undefined;
25+
return {
26+
...actual,
27+
readProcessStartTime: (pid: number) => {
28+
if (pid !== process.pid) return null;
29+
if (cachedOwnStartTime === undefined) {
30+
cachedOwnStartTime = actual.readProcessStartTime(process.pid);
31+
}
32+
return cachedOwnStartTime;
33+
},
34+
};
35+
});
36+
937
test('device status is daemonless and does not send a daemon request', async () => {
1038
const result = await runCliCapture(['device', 'status', '--json']);
1139
assert.equal(result.code, null);

‎src/platforms/apple/core/__tests__/runner-session.test.ts‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ const {
2424
mockIsProcessAlive,
2525
mockIsProcessGroupAlive,
2626
mockPrepareXctestrunWithEnv,
27+
mockReadProcessStartTime,
2728
mockResolveExpectedRunnerCacheMetadata,
2829
mockResolveRunnerDerivedPath,
2930
mockRunAppleToolCommand,
@@ -40,6 +41,12 @@ const {
4041
mockIsProcessAlive: vi.fn(),
4142
mockIsProcessGroupAlive: vi.fn(),
4243
mockPrepareXctestrunWithEnv: vi.fn(),
44+
// Non-empty default: RUNNER_OWNER_START_TIME below is computed at module
45+
// load (before beforeEach), and readProcessStartTime's real implementation
46+
// shells out to `ps` with a 1s timeout that can miss under CPU contention,
47+
// flipping a live owner to 'owner-process-dead'. Deterministic value, no
48+
// shell-out; identity is still enforced by pid in beforeEach below.
49+
mockReadProcessStartTime: vi.fn((_pid: number) => 'fixed-test-owner-start-time' as string | null),
4350
mockResolveExpectedRunnerCacheMetadata: vi.fn(),
4451
mockResolveRunnerDerivedPath: vi.fn(),
4552
mockRunAppleToolCommand: vi.fn(),
@@ -68,6 +75,7 @@ vi.mock('../../../../utils/host-process.ts', async () => {
6875
...actual,
6976
isProcessAlive: mockIsProcessAlive,
7077
isProcessGroupAlive: mockIsProcessGroupAlive,
78+
readProcessStartTime: mockReadProcessStartTime,
7179
};
7280
});
7381

@@ -156,6 +164,12 @@ beforeEach(async () => {
156164
mockRunAppleToolCommand.mockResolvedValue({ exitCode: 0, stdout: '', stderr: '' });
157165
mockIsProcessAlive.mockReturnValue(true);
158166
mockIsProcessGroupAlive.mockReturnValue(false);
167+
// Our pid reads back its fixed start time; any other pid reads as
168+
// not-found, same as a real `ps` miss. Dead-lease tests use fabricated
169+
// pids already rejected by mockIsProcessAlive before this is consulted.
170+
mockReadProcessStartTime.mockImplementation((pid: number) =>
171+
pid === process.pid ? RUNNER_OWNER_START_TIME : null,
172+
);
159173
mockWaitForRunner.mockResolvedValue(runnerResponse({ uptimeMs: 1 }));
160174
});
161175

‎src/utils/__tests__/daemon-client.test.ts‎

Lines changed: 48 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { test } from 'vitest';
1+
import { test, vi } from 'vitest';
22
import assert from 'node:assert/strict';
33
import http from 'node:http';
44
import net from 'node:net';
@@ -36,6 +36,36 @@ import {
3636
import { stopProcessForTakeover } from '../../daemon/daemon-process.ts';
3737
import { findProjectRoot, readVersion } from '../version.ts';
3838

39+
// readProcessStartTime/readProcessCommand shell out to `ps` with a 1s
40+
// timeout (see host-process.ts). isAgentDeviceDaemonProcess re-reads both for
41+
// every liveness check, so a spawned-daemon fixture that is proven live once
42+
// (a real read, right after the process starts) can still be misclassified
43+
// as dead later if a *subsequent* `ps` call happens to miss its deadline
44+
// under full-suite CPU contention. mockReadProcessStartTime/mockReadProcessCommand
45+
// default to `undefined`, which falls through to the real implementation for
46+
// every pid in every test in this file; only the one test below that needs a
47+
// stable answer for its spawned pid configures an override, and clears it
48+
// afterward.
49+
const { mockReadProcessStartTime, mockReadProcessCommand } = vi.hoisted(() => ({
50+
mockReadProcessStartTime: vi.fn<(pid: number) => string | null | undefined>(),
51+
mockReadProcessCommand: vi.fn<(pid: number) => string | null | undefined>(),
52+
}));
53+
54+
vi.mock('../host-process.ts', async () => {
55+
const actual = await vi.importActual<typeof import('../host-process.ts')>('../host-process.ts');
56+
return {
57+
...actual,
58+
readProcessStartTime: (pid: number) => {
59+
const overridden = mockReadProcessStartTime(pid);
60+
return overridden !== undefined ? overridden : actual.readProcessStartTime(pid);
61+
},
62+
readProcessCommand: (pid: number) => {
63+
const overridden = mockReadProcessCommand(pid);
64+
return overridden !== undefined ? overridden : actual.readProcessCommand(pid);
65+
},
66+
};
67+
});
68+
3969
type MockHttpResponse = EventEmitter & {
4070
headers?: Record<string, string>;
4171
statusCode?: number;
@@ -413,11 +443,25 @@ test('cleanupFailedDaemonStartupMetadata retains live startup daemon on timeout'
413443

414444
try {
415445
await new Promise((resolve) => setTimeout(resolve, 50));
446+
// Read the spawned daemon's real identity once (ground truth: it is
447+
// genuinely alive, with this real start time and command line), then
448+
// pin readProcessStartTime/readProcessCommand to keep returning these
449+
// same proven-real values for this pid. isAgentDeviceDaemonProcess reads
450+
// both again internally on every call inside cleanupFailedDaemonStartupMetadata;
451+
// without pinning, a second real `ps` call could miss its 1s timeout
452+
// under load and misclassify this genuinely-live daemon as dead.
416453
const processStartTime = readProcessStartTime(pid) ?? undefined;
417-
if (readProcessCommand(pid) === null || processStartTime === undefined) {
454+
const command = readProcessCommand(pid);
455+
if (command === null || processStartTime === undefined) {
418456
t.skip('process command/start inspection is unavailable in this environment');
419457
return;
420458
}
459+
mockReadProcessStartTime.mockImplementation((queriedPid: number) =>
460+
queriedPid === pid ? processStartTime : undefined,
461+
);
462+
mockReadProcessCommand.mockImplementation((queriedPid: number) =>
463+
queriedPid === pid ? command : undefined,
464+
);
421465

422466
const paths = resolveDaemonPaths(stateDir);
423467
fs.mkdirSync(paths.baseDir, { recursive: true });
@@ -450,6 +494,8 @@ test('cleanupFailedDaemonStartupMetadata retains live startup daemon on timeout'
450494
assert.equal(fs.existsSync(paths.infoPath), true);
451495
assert.equal(fs.existsSync(paths.lockPath), true);
452496
} finally {
497+
mockReadProcessStartTime.mockReset();
498+
mockReadProcessCommand.mockReset();
453499
if (isProcessAlive(pid)) {
454500
process.kill(pid, 'SIGKILL');
455501
await waitForProcessExit(pid, 1_500);

0 commit comments

Comments
 (0)