Skip to content

Commit 84ffcc7

Browse files
committed
test: refuse foreign-pid signals from unit-test workers
A vitest worker may signal only itself and the processes it spawned. src/__tests__/hermetic-signal-setup.ts records any other process.kill, answers it with ESRCH (so best-effort kill paths proceed as if the pid were dead), and fails the sending test by name in afterEach. The senders this catches today are the Apple runner tests, which fabricate runner child pids (4242, 4141, 4343, 4444) and mocked the liveness reads in host-process.ts but not the signal writes: killRunnerProcessTree delivered real SIGINT/SIGTERM/SIGKILL to those pids and their process groups — 146 signals per run of runner-session.test.ts. On the CI runner the sibling vitest forks live in that pid band, so the Coverage job periodically lost one fork mid-file with no test attributed (issue #1824, 6 of the last 40 red CI runs). The group-signal write moves behind signalProcessGroupBestEffort in host-process.ts, next to signalPidsBestEffort, so the runner tests mock the signal seam in the same place they already mock the liveness reads. Refs #1824
1 parent a8a0200 commit 84ffcc7

11 files changed

Lines changed: 289 additions & 107 deletions

‎docs/agents/testing.md‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,16 @@ the runtime seam existed; retiring `dispatchCommand('snapshot')` surfaced them o
100100
time. Do not add to that set, and when a command migrates (`docs/agents/adr-0019-unit.md`), its
101101
tests move to the runtime seam in the same PR.
102102

103+
Signals are hermetic too. A vitest worker may signal only itself and the processes it spawned;
104+
`src/__tests__/hermetic-signal-setup.ts` refuses every other `process.kill` (signal 0, the liveness
105+
probe, stays free) and fails the sending test by name. The refused pid is one the test made up
106+
(`child: { pid: 4242 }`), and on a real host that number can belong to anyone — on CI it periodically
107+
belonged to a sibling fork, which died mid-file with no test attributed ("Worker exited
108+
unexpectedly", #1824). If a test drives a real kill path against a fabricated pid, mock the signal
109+
writes where it already mocks the liveness reads: `signalPidsBestEffort` and
110+
`signalProcessGroupBestEffort` in `src/utils/host-process.ts` (or `vi.spyOn(process, 'kill')`).
111+
Killing a daemon or Metro fixture the test itself spawned is fine — that pid is the worker's own.
112+
103113
Keep tests behavioral. Do not assert shapes or cases TypeScript already proves.
104114

105115
A test added as a regression pin must be shown to fail without the change it pins — vacuity is the
Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
import childProcess from 'node:child_process';
2+
import module from 'node:module';
3+
import util from 'node:util';
4+
import { afterAll, afterEach, expect } from 'vitest';
5+
6+
// Unit tests must be hermetic with respect to the host's process table: a
7+
// worker may signal only itself and the processes it spawned. Anything else is
8+
// a pid the test made up (`child: { pid: 4242 }`), and on a real host that
9+
// number can belong to anyone — on CI it periodically belonged to a sibling
10+
// vitest fork, which died mid-file with no test attributed
11+
// ("Worker exited unexpectedly", #1824). Refusing the signal here, in every
12+
// worker, turns that silent fork death into a named failure of the test that
13+
// sent it, on any host, deterministically.
14+
//
15+
// Signal 0 is a liveness probe (`isProcessAlive`), not a signal; it stays free.
16+
// A negative pid addresses a process group; it is allowed exactly when the
17+
// group leader is a process this worker spawned (`runCmd`'s tree kill).
18+
// Tests that drive a real kill path against a fabricated pid mock the signal
19+
// seam (`signalPidsBestEffort` / `signalProcessGroupBestEffort` in
20+
// `src/utils/host-process.ts`, or `vi.spyOn(process, 'kill')`) the same way
21+
// they already mock the liveness reads.
22+
23+
const ownPids = new Set<number>([process.pid]);
24+
25+
function rememberChild(child: unknown): void {
26+
const pid = (child as { pid?: unknown } | null)?.pid;
27+
if (typeof pid === 'number') ownPids.add(pid);
28+
}
29+
30+
function wrapSpawner<T extends (...args: never[]) => unknown>(real: T): T {
31+
const wrapped = ((...args: Parameters<T>) => {
32+
const child = real(...args);
33+
rememberChild(child);
34+
return child;
35+
}) as unknown as T;
36+
// `util.promisify(execFile)` resolves through the custom-promisify symbol.
37+
const custom = (real as unknown as Record<symbol, unknown>)[util.promisify.custom];
38+
if (custom) Object.defineProperty(wrapped, util.promisify.custom, { value: custom });
39+
return wrapped;
40+
}
41+
42+
for (const name of ['spawn', 'spawnSync', 'fork', 'exec', 'execFile'] as const) {
43+
(childProcess as unknown as Record<string, unknown>)[name] = wrapSpawner(childProcess[name]);
44+
}
45+
// Named ESM imports of the builtin (`import { spawn } from 'node:child_process'`)
46+
// bind to the CJS exports only after they are re-synced.
47+
module.syncBuiltinESMExports();
48+
49+
const refused: string[] = [];
50+
51+
const realKill = process.kill.bind(process);
52+
process.kill = ((pid: number, signal: string | number = 'SIGTERM') => {
53+
if (signal === 0 || ownPids.has(Math.abs(pid))) return realKill(pid, signal as NodeJS.Signals);
54+
const state = expect.getState();
55+
const where = new Error().stack?.split('\n').slice(2, 6).join('\n') ?? '';
56+
refused.push(
57+
`${String(signal)} -> pid ${pid} (${state.currentTestName ?? 'outside a test'})\n${where}`,
58+
);
59+
// Behave like a dead pid: the caller's ESRCH handling runs, no signal leaves
60+
// this worker, and the record above fails the test in afterEach below.
61+
const error = new Error(
62+
`Refusing to send ${String(signal)} to pid ${pid}: this vitest worker did not spawn it (see afterEach failure).`,
63+
) as NodeJS.ErrnoException;
64+
error.code = 'ESRCH';
65+
throw error;
66+
}) as typeof process.kill;
67+
68+
// Best-effort kill sites swallow the ESRCH above, so the refusal is reported
69+
// here, attributed to the test (or file) that caused it, and never lost.
70+
function failOnRefusedSignals(scope: string): void {
71+
if (refused.length === 0) return;
72+
const count = refused.length;
73+
const report = refused.splice(0).join('\n');
74+
throw new Error(
75+
`${scope} tried to send ${count} real signal(s) to a pid this vitest worker did not spawn. ` +
76+
'A unit test may signal only its own children (and their process groups); mock the signal ' +
77+
'seam in src/utils/host-process.ts (signalPidsBestEffort, signalProcessGroupBestEffort) or ' +
78+
`vi.spyOn(process, 'kill') instead.\n${report}`,
79+
);
80+
}
81+
82+
afterEach(() => failOnRefusedSignals('This test'));
83+
afterAll(() => failOnRefusedSignals('This file'));

‎src/__tests__/test-file-size-ratchet.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,8 @@ const PINNED_TEST_FILE_LINES: Readonly<Record<string, number>> = Object.freeze({
3636
'src/__tests__/remote-connection.test.ts': 2973,
3737
'src/daemon/handlers/__tests__/snapshot-handler.test.ts': 2654,
3838
'src/commands/interaction/runtime/settle.test.ts': 2361,
39-
'src/platforms/apple/core/__tests__/runner-session.test.ts': 2083,
4039
'src/daemon/handlers/__tests__/session-replay-runtime-maestro.test.ts': 2031,
40+
'src/platforms/apple/core/__tests__/runner-session.test.ts': 2001,
4141
'src/utils/__tests__/daemon-client.test.ts': 1910,
4242
'src/utils/__tests__/output.test.ts': 1861,
4343
'src/platforms/android/__tests__/snapshot.test.ts': 1660,

‎src/platforms/apple/core/__tests__/runner-disposal.test.ts‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,13 +14,15 @@ const {
1414
mockRunAppleToolCommand,
1515
mockRunXcrun,
1616
mockSignalPidsBestEffort,
17+
mockSignalProcessGroupBestEffort,
1718
} = vi.hoisted(() => ({
1819
mockCleanupTempFile: vi.fn(),
1920
mockIsProcessAlive: vi.fn(),
2021
mockIsProcessGroupAlive: vi.fn(),
2122
mockRunAppleToolCommand: vi.fn(),
2223
mockRunXcrun: vi.fn(),
2324
mockSignalPidsBestEffort: vi.fn(),
25+
mockSignalProcessGroupBestEffort: vi.fn(),
2426
}));
2527

2628
vi.mock('../../../../utils/host-process.ts', async (importOriginal) => {
@@ -30,6 +32,7 @@ vi.mock('../../../../utils/host-process.ts', async (importOriginal) => {
3032
isProcessAlive: mockIsProcessAlive,
3133
isProcessGroupAlive: mockIsProcessGroupAlive,
3234
signalPidsBestEffort: mockSignalPidsBestEffort,
35+
signalProcessGroupBestEffort: mockSignalProcessGroupBestEffort,
3336
};
3437
});
3538

@@ -51,7 +54,6 @@ import { abortRunnerSessionsAndPrepProcesses } from '../runner/runner-disposal.t
5154

5255
beforeEach(() => {
5356
vi.useFakeTimers();
54-
vi.spyOn(process, 'kill').mockImplementation(() => true);
5557
mockIsProcessAlive.mockReturnValue(true);
5658
mockIsProcessGroupAlive.mockReturnValue(false);
5759
mockRunAppleToolCommand.mockResolvedValue({ exitCode: 0, stdout: '', stderr: '' });
@@ -144,9 +146,8 @@ function makeRunnerSession(
144146
}
145147

146148
function runnerSignals(session: RunnerSession): NodeJS.Signals[] {
147-
return vi
148-
.mocked(process.kill)
149-
.mock.calls.filter(([pid]) => pid === -(session.child.pid ?? 0))
149+
return mockSignalProcessGroupBestEffort.mock.calls
150+
.filter(([pid]) => pid === session.child.pid)
150151
.map(([, signal]) => signal as NodeJS.Signals);
151152
}
152153

‎src/platforms/apple/core/__tests__/runner-request-cancellation.test.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ const {
1818
mockRunAppleToolCommand,
1919
mockRunCmdBackground,
2020
mockRunXcrun,
21+
mockSignalPidsBestEffort,
22+
mockSignalProcessGroupBestEffort,
2123
mockWaitForRunner,
2224
mockRedirectRelease,
2325
} = vi.hoisted(() => ({
@@ -32,6 +34,9 @@ const {
3234
mockRunAppleToolCommand: vi.fn(),
3335
mockRunCmdBackground: vi.fn(),
3436
mockRunXcrun: vi.fn(),
37+
// Runner child pids here are fabricated (4141..4444); see runner-session.test.ts.
38+
mockSignalPidsBestEffort: vi.fn(),
39+
mockSignalProcessGroupBestEffort: vi.fn(),
3540
mockWaitForRunner: vi.fn(),
3641
mockRedirectRelease: vi.fn(),
3742
}));
@@ -54,6 +59,8 @@ vi.mock('../../../../utils/host-process.ts', async () => {
5459
...actual,
5560
isProcessAlive: mockIsProcessAlive,
5661
isProcessGroupAlive: mockIsProcessGroupAlive,
62+
signalPidsBestEffort: mockSignalPidsBestEffort,
63+
signalProcessGroupBestEffort: mockSignalProcessGroupBestEffort,
5764
};
5865
});
5966

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
import assert from 'node:assert/strict';
2+
import { EventEmitter } from 'node:events';
3+
import fs from 'node:fs';
4+
import { IOS_SIMULATOR } from '../../../../__tests__/test-utils/index.ts';
5+
import { mkdtempForTestSync } from '../../../../__tests__/test-utils/tmp-dir.ts';
6+
import {
7+
flushDiagnosticsToSessionFile,
8+
withDiagnosticsScope,
9+
} from '../../../../utils/diagnostics.ts';
10+
import { RUNNER_OWNER_START_TIME, type RunnerLease } from '../runner/runner-lease.ts';
11+
import type { RunnerSession } from '../runner/runner-session-types.ts';
12+
13+
// Fabricated runner sessions, leases, background children, and transport
14+
// payloads shared by the runner-session tests. The child pids here are made up
15+
// (`4242`): nothing in a test may deliver a real signal to them, so the owning
16+
// tests mock the signal seam in `src/utils/host-process.ts` — see
17+
// `src/__tests__/hermetic-signal-setup.ts` and #1824.
18+
19+
export function makeRunnerSession(overrides: Partial<RunnerSession> = {}): RunnerSession {
20+
return {
21+
sessionId: `session-${overrides.port ?? 8100}`,
22+
device: IOS_SIMULATOR,
23+
deviceId: IOS_SIMULATOR.id,
24+
port: 8100,
25+
xctestrunPath: '/tmp/runner.xctestrun',
26+
jsonPath: '/tmp/runner.json',
27+
testPromise: Promise.resolve({ exitCode: 0, stdout: '', stderr: '' }),
28+
child: { pid: 1234, exitCode: null },
29+
ready: true,
30+
...overrides,
31+
} as RunnerSession;
32+
}
33+
34+
export function makeRunnerLease(
35+
overrides: Partial<RunnerLease> & { deviceId: string; ownerToken?: string | undefined },
36+
): RunnerLease {
37+
const ownerToken = overrides.ownerToken ?? `owner-${process.pid}-test`;
38+
const lease: RunnerLease = {
39+
schemaVersion: 1,
40+
deviceId: overrides.deviceId,
41+
ownerToken,
42+
ownerPid: process.pid,
43+
ownerStartTime: RUNNER_OWNER_START_TIME,
44+
sessionId: `session-${overrides.deviceId}`,
45+
runnerPid: 4242,
46+
port: 8123,
47+
xctestrunPath: `/tmp/AgentDeviceRunner.env.session-${overrides.deviceId}-${ownerToken}-8123.xctestrun`,
48+
jsonPath: `/tmp/AgentDeviceRunner.env.session-${overrides.deviceId}-${ownerToken}-8123.json`,
49+
createdAtMs: Date.now(),
50+
};
51+
return { ...lease, ...overrides, ownerToken };
52+
}
53+
54+
export function makeBackgroundRunner(pid: number) {
55+
return {
56+
child: {
57+
pid,
58+
exitCode: null,
59+
stdout: new EventEmitter(),
60+
stderr: new EventEmitter(),
61+
},
62+
wait: Promise.resolve({ exitCode: 0, stdout: '', stderr: '' }),
63+
};
64+
}
65+
66+
export function runnerResponse(data: Record<string, unknown>): Response {
67+
return new Response(JSON.stringify({ ok: true, data }));
68+
}
69+
70+
export function runnerError(error: { code: string; message: string }): Response {
71+
return new Response(JSON.stringify({ ok: false, error }));
72+
}
73+
74+
export async function captureDiagnostics(callback: () => Promise<void>): Promise<string> {
75+
const previousHome = process.env.HOME;
76+
process.env.HOME = mkdtempForTestSync('agent-device-runner-diag-');
77+
try {
78+
return await withDiagnosticsScope(
79+
{ session: 'runner-session-test', requestId: 'request-1', command: 'tap' },
80+
async () => {
81+
await callback();
82+
const diagnosticsPath = flushDiagnosticsToSessionFile({ force: true })?.path;
83+
assert.ok(diagnosticsPath);
84+
return fs.readFileSync(diagnosticsPath, 'utf8');
85+
},
86+
);
87+
} finally {
88+
process.env.HOME = previousHome;
89+
}
90+
}
91+
92+
export function assertRunnerCommand(
93+
actual: unknown,
94+
expected: Record<string, unknown>,
95+
options: { commandId?: boolean } = {},
96+
): asserts actual is Record<string, unknown> {
97+
assert.equal(typeof actual, 'object');
98+
assert.notEqual(actual, null);
99+
const command = actual as Record<string, unknown>;
100+
const commandId = command.commandId;
101+
if (options.commandId === false) {
102+
assert.equal(commandId, undefined);
103+
assert.deepEqual(command, expected);
104+
return;
105+
}
106+
if (typeof commandId !== 'string') {
107+
assert.fail('expected runner commandId');
108+
}
109+
assert.match(commandId, /^runner-/);
110+
assert.deepEqual({ ...command, commandId: undefined }, { ...expected, commandId: undefined });
111+
}

0 commit comments

Comments
 (0)