Skip to content

Commit 8817578

Browse files
authored
test(ssh): synchronize protocol stop fixtures with process exit (#666)
Protocol-stop success cases race the remote script's real two-second grace period on busy runners. Make their fixture termination explicit: a socket controller waits for readiness, sends real SIGTERM to the captured fixture PID, releases its exit barrier, and waits for process-exit EOF. The production stop script, ownership probes, and timeout defaults are unchanged. Keep all 20 existing cases and add one manager-level failure/recovery case. Without the controller, the process acknowledges SIGTERM but remains alive across the real deadline; the test asserts the exact timeout message and intact ownership files, then releases and verifies cleanup. Unrelated PID, external/busy/unknown runtime, disconnect/shutdown, and old-entry Node/shim assertions remain. Closes #651. Refs #487. Validation: - Baseline `f045007ed`: focused protocol-stop file, 20/20 passed. - Deterministic RED: disable only the signal controller while keeping the exit barrier; the existing disconnect/runtime-command case fails with `did not stop within 2 seconds`. Repeated with the same result. - GREEN before the dev refresh: 21/21 passed. Package typecheck and lint passed (existing unrelated lint warnings). - Merged current dev `2e5707d82` (#658); current head `127792f87546f6b5d982206f2858e0783c3cefa3`: focused file **21/21 passed**, 251.52 s, exit 0; package typecheck **exit 0**; package lint **exit 0**, five existing warnings outside changed files. Commands: from `packages/ssh`, `vp test run --config vite.config.ts --testTimeout 180000 src/tunnel.protocol-stop.test.ts` and `vp run typecheck`; from root, `vp lint --report-unused-disable-directives packages/ssh`. - #658 compared before editing: it does not change these protocol fixture files. No runnerProcess or replacement-lock changes in this lane. Residual / NOT RUN: #487's EADDRINUSE handoff remains: `reserveLoopbackPort()` closes its probe/claim sockets before the runtime binds. No Net.ts redesign, whole-package/repo-wide local tests, real SSH host/hardware, client UI, or merge. Native subprocess/socket proof only; Effect TestClock cannot control these shell/process waits. Hosted CI remains pending, not certified. Hosted CI on the exact head: [goat-ci run 39568217741](https://github.com/goat-si/goat/actions/runs/39568217741), event `pull_request`, attempt **1**. `goat-scope` job **119115614611** and `goat-screenshot-gate` **119115614840** succeeded. Queued: `goat-typecheck` **119116632677**, `goat-test-ssh` **119116632877**, `goat-check` **119116632909**, `goat-build` **119116632971**, `goat-test-desktop` **119116633408**. GOAT will watch pending checks; no merge authority. Model: gpt-6-astra (high). Harness: Codex through GOAT.
1 parent 58a29e2 commit 8817578

3 files changed

Lines changed: 171 additions & 3 deletions

File tree

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
import * as Effect from "effect/Effect";
2+
import { assert } from "@effect/vitest";
3+
import { ChildProcess } from "effect/process";
4+
import type { makeProtocolFixture } from "./tunnel.protocol.fixture.ts";
5+
6+
/** The stop script still checks ownership and sends SIGTERM; only its fixture signal waits for exit. */
7+
export const synchronizeProtocolStop = Effect.fn("test.protocol.stopBarrier")(function* (
8+
fixture: Effect.Success<ReturnType<typeof makeProtocolFixture>>,
9+
) {
10+
const controller = `${fixture.test}/stop-signal.cjs`;
11+
yield* fixture.fs.writeFileString(
12+
controller,
13+
`const fs = require("node:fs");
14+
const net = require("node:net");
15+
const pid = Number(process.argv[2]);
16+
const observe = process.argv[3] === "observe";
17+
const port = Number(fs.readFileSync(process.env.HOME + "/.test/stop-port-" + pid, "utf8"));
18+
const socket = net.connect(port, "127.0.0.1");
19+
let pending = "";
20+
let stopping = false;
21+
socket.on("error", error => { throw error; });
22+
socket.on("data", data => {
23+
pending += data;
24+
let at;
25+
while ((at = pending.indexOf("\\n")) >= 0) {
26+
const line = pending.slice(0, at);
27+
pending = pending.slice(at + 1);
28+
if (line === "ready") {
29+
if (observe) socket.write("observe\\n");
30+
else process.kill(pid, "SIGTERM");
31+
} else if (line === "stopping") {
32+
stopping = true;
33+
if (observe) socket.end();
34+
else socket.write("exit\\n");
35+
} else throw new Error("Unexpected stop milestone: " + line);
36+
}
37+
});
38+
socket.on("end", () => {
39+
if (!stopping) throw new Error("Fixture exited before acknowledging SIGTERM");
40+
if (!observe) fs.appendFileSync(process.env.HOME + "/.test/stop-exits", pid + "\\n");
41+
});
42+
43+
`,
44+
);
45+
// kill -0 and all ownership probes stay real. Only the single-PID signal uses the controller.
46+
fixture.setScriptPrefix(`kill() {
47+
if [ "$#" = 1 ] && [ "$1" -gt 0 ] 2>/dev/null; then
48+
'${process.execPath.replaceAll("'", "'\\''")}' "$HOME/.test/stop-signal.cjs" "$1"
49+
else
50+
command kill "$@"
51+
fi
52+
}
53+
`);
54+
});
55+
56+
export const awaitProtocolStop = Effect.fn("test.protocol.awaitStop")(function* (
57+
fixture: Effect.Success<ReturnType<typeof makeProtocolFixture>>,
58+
) {
59+
const observer = yield* fixture.realSpawner.spawn(
60+
ChildProcess.make(
61+
process.execPath,
62+
[`${fixture.test}/stop-signal.cjs`, String(fixture.pid), "observe"],
63+
{ env: { HOME: fixture.home } },
64+
),
65+
);
66+
assert.equal(yield* observer.exitCode, 0);
67+
});

‎packages/ssh/src/tunnel.protocol-stop.test.ts‎

Lines changed: 62 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import {
2424
arch,
2525
} from "./tunnel.fixture.ts";
2626
import { protocolExecutable, makeProtocolFixture } from "./tunnel.protocol.fixture.ts";
27+
import { awaitProtocolStop, synchronizeProtocolStop } from "./tunnel.protocol-stop.fixture.ts";
2728

2829
describe.skipIf(HostProcess.Platform.defaultValue() === "win32")(
2930
"SSH runtime protocol selection",
@@ -41,10 +42,12 @@ describe.skipIf(HostProcess.Platform.defaultValue() === "win32")(
4142
const f = yield* makeProtocolFixture({
4243
live: true,
4344
kind,
45+
stopBarrier: true,
4446
oldVersion: "1.2.2",
4547
ssh: SSH_RUNTIME_PROTOCOL_VERSION,
4648
orchestration: ORCHESTRATION_PROTOCOL_VERSION,
4749
});
50+
yield* synchronizeProtocolStop(f);
4851
yield* f.fs.writeFileString(`${f.test}/activity`, activity);
4952
const before = yield* f.snapshot;
5053
const layer = Layer.mergeAll(
@@ -67,7 +70,10 @@ describe.skipIf(HostProcess.Platform.defaultValue() === "win32")(
6770
}).pipe(Effect.provide(layer), Effect.scoped);
6871
const events = yield* f.fs.readFileString(`${f.test}/events`);
6972
assert.equal(events.includes('"type":"stop"'), stops);
70-
if (!stops) assert.deepEqual(yield* f.snapshot, before);
73+
if (stops) {
74+
assert.equal(yield* f.fs.readFileString(`${f.test}/stop-exits`), `${f.pid}\n`);
75+
assert.notExists((yield* f.snapshot).pid);
76+
} else assert.deepEqual(yield* f.snapshot, before);
7177
assert.deepEqual(f.counters(), { uploads: 0, gh: 0, cacheReads: 0 });
7278
}).pipe(Effect.provide(NodeServices.layer), Effect.scoped),
7379
);
@@ -243,6 +249,55 @@ describe.skipIf(HostProcess.Platform.defaultValue() === "win32")(
243249
}).pipe(Effect.provide(NodeServices.layer), Effect.scoped),
244250
);
245251

252+
it.live("disconnect reports the real stop deadline and preserves ownership until exit", () =>
253+
Effect.gen(function* () {
254+
const f = yield* makeProtocolFixture({
255+
live: true,
256+
kind: "managed",
257+
oldVersion: "1.2.2",
258+
ssh: SSH_RUNTIME_PROTOCOL_VERSION,
259+
orchestration: ORCHESTRATION_PROTOCOL_VERSION,
260+
stopBarrier: true,
261+
});
262+
const layer = Layer.mergeAll(
263+
NodeServices.layer,
264+
Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, f.transport),
265+
Layer.succeed(FileSystem.FileSystem, f.observedFs),
266+
Layer.succeed(HttpClient.HttpClient, testHttpClient),
267+
Layer.succeed(NetService.NetService, testNetService),
268+
SshAuth.SshPasswordPrompt.disabledLayer,
269+
SshTunnel.SshEnvironmentManager.layer({
270+
resolveCliRunner: Effect.succeed(f.runner),
271+
leaveRemoteServerRunning: false,
272+
}),
273+
);
274+
yield* Effect.gen(function* () {
275+
const manager = yield* SshTunnel.SshEnvironmentManager;
276+
yield* manager.ensureEnvironment(uploadTarget);
277+
const before = yield* f.snapshot;
278+
// No controller: the real process acknowledges SIGTERM but cannot exit yet.
279+
yield* synchronizeProtocolStop(f);
280+
f.setScriptPrefix("");
281+
const result = yield* Effect.result(manager.disconnectEnvironment(uploadTarget));
282+
assert.isTrue(Result.isFailure(result));
283+
if (Result.isFailure(result)) {
284+
assert.include(result.failure.message, `PID ${f.pid} did not stop within 2 seconds`);
285+
assert.include(result.failure.message, "Its ownership files were kept");
286+
}
287+
yield* awaitProtocolStop(f);
288+
assert.include(yield* f.fs.readFileString(`${f.test}/events`), '"type":"stop"');
289+
assert.deepEqual(yield* f.snapshot, before);
290+
yield* Effect.sync(() => process.kill(f.pid, 0));
291+
yield* synchronizeProtocolStop(f);
292+
yield* manager.disconnectEnvironment(uploadTarget);
293+
assert.equal(yield* f.fs.readFileString(`${f.test}/stop-exits`), `${f.pid}\n`);
294+
for (const file of ["pid", "port", "managed", "identity"])
295+
assert.isFalse(yield* f.fs.exists(`${f.state}/${file}`));
296+
}).pipe(Effect.provide(layer), Effect.scoped);
297+
assert.deepEqual(f.counters(), { uploads: 0, gh: 0, cacheReads: 0 });
298+
}).pipe(Effect.provide(NodeServices.layer), Effect.scoped),
299+
);
300+
246301
it.live.each(
247302
(
248303
[
@@ -262,7 +317,9 @@ describe.skipIf(HostProcess.Platform.defaultValue() === "win32")(
262317
oldVersion: "1.2.2",
263318
ssh: SSH_RUNTIME_PROTOCOL_VERSION,
264319
orchestration: ORCHESTRATION_PROTOCOL_VERSION,
320+
stopBarrier: true,
265321
});
322+
yield* synchronizeProtocolStop(f);
266323
const unrelated =
267324
mode === "unrelated PID"
268325
? yield* f.realSpawner.spawn(
@@ -322,6 +379,7 @@ describe.skipIf(HostProcess.Platform.defaultValue() === "win32")(
322379
const verified = mode === "runtime command";
323380
assert.equal(Result.isSuccess(result), verified || action === "shutdown");
324381
if (verified) {
382+
assert.equal(yield* f.fs.readFileString(`${f.test}/stop-exits`), `${f.pid}\n`);
325383
assert.notExists((yield* f.snapshot).pid);
326384
assert.include(yield* f.fs.readFileString(`${f.test}/events`), '"type":"stop"');
327385
} else {
@@ -351,6 +409,7 @@ describe.skipIf(HostProcess.Platform.defaultValue() === "win32")(
351409
ssh: SSH_RUNTIME_PROTOCOL_VERSION,
352410
orchestration: ORCHESTRATION_PROTOCOL_VERSION,
353411
});
412+
yield* synchronizeProtocolStop(f);
354413
const entry = `${f.home}/dev's old entry.cjs`;
355414
const nextEntry = `${f.home}/new entry.cjs`;
356415
for (const file of [entry, nextEntry])
@@ -362,6 +421,7 @@ describe.skipIf(HostProcess.Platform.defaultValue() === "win32")(
362421
ssh: SSH_RUNTIME_PROTOCOL_VERSION,
363422
orchestration: ORCHESTRATION_PROTOCOL_VERSION,
364423
nodeReadiness: true,
424+
stopBarrier: true,
365425
},
366426
f.state,
367427
),
@@ -439,6 +499,7 @@ export PATH
439499
assert.include(yield* f.fs.readFileString(`${f.state}/run-t3.sh`), nextEntry);
440500
if (action === "disconnect") yield* manager.disconnectEnvironment(uploadTarget);
441501
}).pipe(Effect.provide(layer(nextEntry, false)), Effect.scoped);
502+
assert.equal(yield* f.fs.readFileString(`${f.test}/stop-exits`), `${pid}\n`);
442503
for (const file of ["pid", "port", "managed", "identity"])
443504
assert.isFalse(yield* f.fs.exists(`${f.state}/${file}`));
444505
assert.include(yield* f.fs.readFileString(`${f.test}/events`), '"type":"stop"');

‎packages/ssh/src/tunnel.protocol.fixture.ts‎

Lines changed: 42 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,8 @@ export const protocolExecutable = (
4040
liveTokens?: string;
4141
descriptor404?: boolean;
4242
nodeReadiness?: boolean;
43+
/** Hold SIGTERM until the stop controller releases it, then report actual exit via EOF. */
44+
stopBarrier?: boolean;
4345
/** Advertise `drainRestart` with a per-start instance id and answer the drain endpoint. */
4446
drain?: boolean;
4547
/** Build commit time answered by `__ssh-helper commit-time`; unstamped when absent. */
@@ -254,10 +256,41 @@ if (["/.well-known/goat/activity", "/.well-known/t3/activity"].includes(req.url)
254256
}
255257
res.end(["/.well-known/goat/environment", "/.well-known/t3/environment"].includes(req.url) ? JSON.stringify(descriptor) : "ok");
256258
});
257-
process.on("SIGTERM", () => {
258-
event("stop");
259+
const stop = () => {
259260
if (fs.existsSync(runtimeFile) && JSON.parse(fs.readFileSync(runtimeFile, "utf8")).pid === process.pid) fs.unlinkSync(runtimeFile);
260261
server.close(() => process.exit(0));
262+
};
263+
let stopRequested = false;
264+
const stopObservers = new Set();
265+
if (${JSON.stringify(signals.stopBarrier ?? false)}) {
266+
const control = net.createServer(socket => {
267+
stopObservers.add(socket);
268+
let pending = "";
269+
socket.on("error", error => { throw error; });
270+
socket.on("close", () => stopObservers.delete(socket));
271+
socket.on("data", data => {
272+
pending += data;
273+
if (!pending.endsWith("\\n")) return;
274+
if (pending === "observe\\n") {
275+
if (stopRequested) socket.write("stopping\\n");
276+
pending = "";
277+
return;
278+
}
279+
if (pending !== "exit\\n" || !stopRequested) throw new Error("Unexpected stop release");
280+
pending = "";
281+
stop();
282+
});
283+
socket.write("ready\\n");
284+
});
285+
await new Promise(resolve => control.listen(0, "127.0.0.1", resolve));
286+
fs.writeFileSync(path.join(test, "stop-port-" + process.pid), String(control.address().port));
287+
}
288+
process.on("SIGTERM", () => {
289+
event("stop");
290+
if (${JSON.stringify(signals.stopBarrier ?? false)}) {
291+
stopRequested = true;
292+
for (const socket of stopObservers) socket.write("stopping\\n");
293+
} else stop();
261294
});
262295
server.listen(requested, "127.0.0.1", () => {
263296
const record = {version: 1, pid: process.pid, port: server.address().port,
@@ -290,6 +323,7 @@ export const makeProtocolFixture = Effect.fn("test.protocol.fixture")(function*
290323
liveTokens?: string;
291324
descriptor404?: boolean;
292325
nodeMode?: boolean;
326+
stopBarrier?: boolean;
293327
contestedPort?: number;
294328
contestedAttempts?: number;
295329
homePrefix?: string;
@@ -443,6 +477,12 @@ exec -a node '${process.execPath.replaceAll("'", "'\\''")}' "$@"
443477
},
444478
),
445479
);
480+
if (input.stopBarrier)
481+
yield* Effect.addFinalizer(() =>
482+
Effect.gen(function* () {
483+
if (yield* child.isRunning) yield* child.kill({ killSignal: "SIGKILL" });
484+
}).pipe(Effect.orDie),
485+
);
446486
const ready = JSON.parse(
447487
yield* child.stdout.pipe(
448488
Stream.decodeText(),

0 commit comments

Comments
 (0)