Skip to content

Commit a8a0200

Browse files
thymikeethymikee
committed
ci: w3-1824 experiment — trace fork signals and plant pid sentinels in the Coverage job
Temporary instrumentation for #1824. Every vitest fork logs each real process.kill it sends to a foreign pid (and every kill/pkill it spawns); the Coverage job parks sentinel processes on the pids the Apple runner tests fabricate (4141/4242/4343/4444) and reports which of them survive the run. Reverted before this PR leaves draft.
1 parent fda81c5 commit a8a0200

6 files changed

Lines changed: 227 additions & 1 deletion

File tree

‎.github/workflows/ci.yml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -338,12 +338,22 @@ jobs:
338338
uses: ./.github/actions/run-gate
339339
with: { gate: coverage-model }
340340

341+
# w3-1824 EXPERIMENT (temporary): sentinels on the pids the runner tests
342+
# fabricate, plus a per-fork kill tracer wired through vitest.config.ts.
343+
- name: w3-1824 plant sentinels
344+
run: bash ./w3-1824-plant-sentinels.sh /tmp/w3-1824
345+
341346
- name: Run coverage
342347
id: run-coverage
343348
env:
344349
OUTPUT_ECONOMY_BASE: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event.before }}
350+
W3_1824_KILL_TRACE: /tmp/w3-1824/kill-trace.ndjson
345351
uses: ./.github/actions/run-gate
346352
with: { gate: unit-ci }
353+
354+
- name: w3-1824 report
355+
if: always()
356+
run: bash ./w3-1824-report.sh /tmp/w3-1824 /tmp/w3-1824/kill-trace.ndjson
347357
# The TMPDIR redirection both test lanes depend on (#1593/#1595). The check is a real
348358
# package script that no workflow ran: it is reachable only through `check:unit`, an
349359
# aggregate CI never invokes, so a leak regression could not fail a PR. Placed here

‎vitest.config.ts‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,12 @@ export const SUBPROCESS_STUB_TESTS: readonly string[] = [
1717
'scripts/fuzz/corpus-replay.test.ts',
1818
];
1919

20-
const SETUP_FILES = ['src/__tests__/hermetic-env-setup.ts', 'src/__tests__/process-memo-setup.ts'];
20+
const SETUP_FILES = [
21+
'src/__tests__/hermetic-env-setup.ts',
22+
'src/__tests__/process-memo-setup.ts',
23+
// w3-1824 EXPERIMENT (temporary): per-fork kill tracer.
24+
'./w3-1824-kill-trace-setup.ts',
25+
];
2126

2227
export default defineConfig({
2328
test: {

‎w3-1824-kill-trace-setup.ts‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
// fallow-ignore-file unused-file
2+
// SCRATCH (w3-1824): traces every real signal a vitest fork sends to a foreign
3+
// pid, plus every `kill`/`pkill` subprocess it spawns. Appends NDJSON lines to
4+
// $W3_1824_KILL_TRACE. Never committed.
5+
import childProcess from 'node:child_process';
6+
import fs from 'node:fs';
7+
import module from 'node:module';
8+
import util from 'node:util';
9+
import { beforeAll, expect } from 'vitest';
10+
11+
const out = process.env.W3_1824_KILL_TRACE;
12+
13+
function currentTest(): { file?: string; name?: string } {
14+
try {
15+
const state = expect.getState();
16+
return { file: state.testPath, name: state.currentTestName };
17+
} catch {
18+
return {};
19+
}
20+
}
21+
22+
function readPgid(): number | undefined {
23+
try {
24+
// /proc/self/stat: pid (comm) state ppid pgrp ...
25+
const stat = fs.readFileSync('/proc/self/stat', 'utf8');
26+
const afterComm = stat.slice(stat.lastIndexOf(')') + 2).split(' ');
27+
return Number(afterComm[2]);
28+
} catch {
29+
return undefined;
30+
}
31+
}
32+
33+
function record(entry: Record<string, unknown>): void {
34+
if (!out) return;
35+
const line = JSON.stringify({
36+
ts: new Date().toISOString(),
37+
workerPid: process.pid,
38+
workerPpid: process.ppid,
39+
workerPgid: readPgid(),
40+
...currentTest(),
41+
...entry,
42+
});
43+
fs.appendFileSync(out, `${line}\n`);
44+
}
45+
46+
const realKill = process.kill.bind(process);
47+
process.kill = ((pid: number, signal?: string | number) => {
48+
// signal 0 is a liveness probe; ignore self-signals.
49+
if (signal !== 0 && pid !== process.pid) {
50+
record({
51+
kind: 'process.kill',
52+
pid,
53+
signal: signal ?? 'SIGTERM',
54+
stack: new Error().stack?.split('\n').slice(2, 9).join(' | '),
55+
});
56+
}
57+
return realKill(pid, signal as never);
58+
}) as typeof process.kill;
59+
60+
const KILL_BINARIES = new Set(['kill', 'pkill', 'killall']);
61+
62+
function traceSpawn(cmd: unknown, args: unknown): void {
63+
const command = String(cmd);
64+
if (!KILL_BINARIES.has(command.split('/').pop() ?? '')) return;
65+
record({
66+
kind: 'spawn',
67+
command,
68+
args: Array.isArray(args) ? args : [],
69+
stack: new Error().stack?.split('\n').slice(3, 10).join(' | '),
70+
});
71+
}
72+
73+
for (const name of ['spawn', 'spawnSync', 'execFile', 'execFileSync'] as const) {
74+
const real = childProcess[name] as (...a: unknown[]) => unknown;
75+
const wrapped = (...a: unknown[]) => {
76+
traceSpawn(a[0], a[1]);
77+
return real(...a);
78+
};
79+
// util.promisify(execFile) relies on the custom promisify symbol; keep it.
80+
const custom = (real as unknown as Record<symbol, unknown>)[util.promisify.custom];
81+
if (custom) Object.defineProperty(wrapped, util.promisify.custom, { value: custom });
82+
(childProcess as unknown as Record<string, unknown>)[name] = wrapped;
83+
}
84+
85+
// Named ESM imports of the builtin (`import { spawn } from 'node:child_process'`)
86+
// only see the patched functions after the CJS exports are re-synced.
87+
module.syncBuiltinESMExports();
88+
record({ kind: 'worker-start' });
89+
beforeAll(() => {
90+
record({ kind: 'file-start' });
91+
});

‎w3-1824-plant-sentinels.sh‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
#!/usr/bin/env bash
2+
# SCRATCH (w3-1824): park a sentinel process on each pid the runner-session and
3+
# runner-request-cancellation tests fabricate (4141, 4242, 4343, 4444). Linux
4+
# allocates pids sequentially, so the script spawns throwaway subshells until
5+
# the counter sits just below each target and then starts the sentinel there.
6+
# No privileges, no kernel knobs: just process creation.
7+
set -euo pipefail
8+
LOG_DIR="${1:?log dir}"
9+
mkdir -p "$LOG_DIR"
10+
echo "nproc=$(nproc) mem=$(free -m | awk '/Mem:/ {print $2}')MB pid_max=$(cat /proc/sys/kernel/pid_max)"
11+
echo "step shell: pid/ppid/pgid/sid = $(ps -o pid=,ppid=,pgid=,sid= -p $$)"
12+
SENTINEL="$(cd "$(dirname "$0")" && pwd)/w3-1824-sentinel.cjs"
13+
14+
for target in 4141 4242 4343 4444; do
15+
if kill -0 "$target" 2>/dev/null; then
16+
echo "pid $target already taken by: $(ps -o cmd= -p "$target")"
17+
continue
18+
fi
19+
spawned=0
20+
while :; do
21+
( : ) &
22+
p=$!
23+
wait "$p" || true
24+
spawned=$((spawned + 1))
25+
if [ "$p" -eq $((target - 1)) ]; then break; fi
26+
if [ "$p" -gt $((target - 1)) ]; then
27+
echo "missed $target (counter already at $p after $spawned spawns)"
28+
p=""
29+
break
30+
fi
31+
if [ "$spawned" -gt 20000 ]; then
32+
echo "gave up on $target after $spawned spawns (counter at $p)"
33+
p=""
34+
break
35+
fi
36+
done
37+
[ -n "$p" ] || continue
38+
node "$SENTINEL" "$LOG_DIR/sentinel-$target.log" &
39+
pid=$!
40+
if [ "$pid" -eq "$target" ]; then
41+
echo "sentinel planted at pid $pid after $spawned spawns: $(ps -o pid=,ppid=,pgid=,sid=,cmd= -p "$pid")"
42+
echo "$pid" >> "$LOG_DIR/sentinels.txt"
43+
else
44+
echo "sentinel landed on $pid instead of $target; killing it"
45+
kill "$pid" 2>/dev/null || true
46+
fi
47+
done
48+
disown -a 2>/dev/null || true
49+
echo "sentinels: $(cat "$LOG_DIR/sentinels.txt" 2>/dev/null | tr '\n' ' ')"

‎w3-1824-report.sh‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
#!/usr/bin/env bash
2+
# SCRATCH (w3-1824): after the coverage run, report which sentinels survived,
3+
# what signals they saw, whether the kernel OOM-killed anything, and every real
4+
# signal a vitest fork sent to a foreign pid (from the kill tracer).
5+
set -uo pipefail
6+
LOG_DIR="${1:?log dir}"
7+
TRACE="${2:?trace file}"
8+
9+
echo "== sentinels"
10+
if [ -f "$LOG_DIR/sentinels.txt" ]; then
11+
while read -r pid; do
12+
if kill -0 "$pid" 2>/dev/null; then
13+
echo "pid $pid: ALIVE ($(ps -o cmd= -p "$pid" | cut -c1-80))"
14+
else
15+
echo "pid $pid: DEAD"
16+
fi
17+
cat "$LOG_DIR/sentinel-$pid.log" 2>/dev/null | sed 's/^/ /'
18+
done < "$LOG_DIR/sentinels.txt"
19+
else
20+
echo "(none planted)"
21+
fi
22+
23+
echo "== kernel OOM / kill lines"
24+
(sudo dmesg 2>/dev/null || dmesg 2>/dev/null) | grep -iE 'out of memory|oom-kill|killed process' || echo "(none)"
25+
26+
echo "== kill trace"
27+
if [ -f "$TRACE" ]; then
28+
node - "$TRACE" <<'EOF'
29+
const fs = require('node:fs');
30+
const lines = fs.readFileSync(process.argv[2], 'utf8').trim().split('\n').filter(Boolean).map((l) => JSON.parse(l));
31+
const rel = (f) => (f || '?').replace(/.*\/agent-device\/agent-device\//, '');
32+
const starts = lines.filter((e) => e.kind === 'file-start');
33+
console.log(`${lines.length} events, ${starts.length} files; worker pids ${Math.min(...starts.map((e) => e.workerPid))}..${Math.max(...starts.map((e) => e.workerPid))}; pgid(s) ${[...new Set(starts.map((e) => e.workerPgid))].join(',')}`);
34+
const firstUnit = starts.find((e) => rel(e.file).startsWith('src/'));
35+
if (firstUnit) console.log(`first src/ file ${rel(firstUnit.file)} started at ${firstUnit.ts} in worker pid ${firstUnit.workerPid}`);
36+
const kills = lines.filter((e) => e.kind === 'process.kill' || e.kind === 'spawn');
37+
const byFile = new Map();
38+
for (const e of kills) {
39+
const key = rel(e.file);
40+
const what = e.kind === 'spawn' ? `spawn ${e.command} ${e.args.join(' ')}` : `kill ${e.pid} ${e.signal}`;
41+
const m = byFile.get(key) ?? new Map();
42+
m.set(what, (m.get(what) ?? 0) + 1);
43+
byFile.set(key, m);
44+
}
45+
for (const [file, m] of byFile) {
46+
console.log(`-- ${file}`);
47+
for (const [what, n] of m) console.log(` ${n}x ${what}`);
48+
}
49+
const forkPids = new Set(starts.map((e) => e.workerPid));
50+
const hits = kills.filter((e) => e.kind === 'process.kill' && forkPids.has(Math.abs(e.pid)));
51+
console.log(`== signals aimed at a pid that was also a vitest fork in this run: ${hits.length}`);
52+
for (const e of hits) console.log(` ${e.ts} ${rel(e.file)} -> kill ${e.pid} ${e.signal} (fork ran ${rel(starts.find((s) => s.workerPid === Math.abs(e.pid))?.file)})`);
53+
EOF
54+
else
55+
echo "(no trace file at $TRACE)"
56+
fi

‎w3-1824-sentinel.cjs‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
// fallow-ignore-file unused-file
2+
// SCRATCH (w3-1824): a process that sits at a chosen pid and records which
3+
// signal reached it. SIGKILL cannot be logged; its death is the record.
4+
const fs = require('node:fs');
5+
const out = process.argv[2];
6+
const log = (line) =>
7+
fs.appendFileSync(out, `${new Date().toISOString()} pid=${process.pid} ${line}\n`);
8+
for (const signal of ['SIGINT', 'SIGTERM', 'SIGHUP']) {
9+
process.on(signal, () => {
10+
log(`received ${signal}`);
11+
process.exit(0);
12+
});
13+
}
14+
log('started');
15+
setInterval(() => {}, 1_000_000);

0 commit comments

Comments
 (0)