Skip to content

Commit d3258a2

Browse files
committed
fix(ssh): preserve live remote servers when reconnecting
1 parent 38c187b commit d3258a2

2 files changed

Lines changed: 165 additions & 59 deletions

File tree

‎packages/ssh/src/tunnel.test.ts‎

Lines changed: 143 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -244,7 +244,7 @@ describe("ssh tunnel scripts", () => {
244244
launch,
245245
'[ -n "$REMOTE_PID" ] && [ -n "$REMOTE_PORT" ] && kill -0 "$REMOTE_PID" 2>/dev/null',
246246
);
247-
assert.include(launch, "RUNNER_CHANGED=1");
247+
assert.notInclude(launch, "RUNNER_CHANGED");
248248
assert.include(launch, "ensure_remote_node_path()");
249249
assert.include(launch, "if ! ensure_remote_node_path; then");
250250
assert.include(devLaunch, `T3_NODE_ENGINE_RANGE='${TEST_NODE_ENGINE_RANGE}'`);
@@ -291,19 +291,11 @@ describe("ssh tunnel scripts", () => {
291291
assert.include(launch, "resolve_default_runtime_port()");
292292
assert.include(launch, 'DEFAULT_RUNTIME_INFO="$(resolve_default_runtime_port');
293293
assert.include(launch, "if (!Number.isInteger(pid) || pid <= 0 || !Number.isInteger(port))");
294-
assert.include(launch, 'PID_TO_STOP="${REMOTE_PID:-$DEFAULT_RUNTIME_PID}"');
294+
assert.notInclude(launch, "PID_TO_STOP");
295295
assert.include(launch, 'REMOTE_PORT="$DEFAULT_REMOTE_PORT"');
296296
assert.include(launch, 'rm -f "$PID_FILE"');
297297
assert.include(launch, "printf 'external\\n' >\"$MANAGED_FILE\"");
298298
assert.include(launch, 'if [ -z "$REMOTE_PORT" ]; then');
299-
assert.isBelow(
300-
launch.indexOf('if [ "$REMOTE_MANAGED" = "managed" ]'),
301-
launch.indexOf("printf 'external\\n' >\"$MANAGED_FILE\""),
302-
);
303-
assert.isBelow(
304-
launch.indexOf('DEFAULT_RUNTIME_INFO="$(resolve_default_runtime_port'),
305-
launch.indexOf('elif [ -n "$REMOTE_PID" ]'),
306-
);
307299
});
308300

309301
it.effect("accepts launch JSON after remote shell startup noise", () => {
@@ -484,6 +476,59 @@ describe("ssh tunnel scripts", () => {
484476
}).pipe(Effect.provide(layerProcess));
485477
});
486478

479+
it.effect("replaces a stale SSH forward without stopping the remote server", () =>
480+
Effect.gen(function* () {
481+
const probeStarted = yield* Deferred.make<void>();
482+
let stale = false;
483+
let forwards = 0;
484+
let stops = 0;
485+
let kills = 0;
486+
const spawner = ChildProcessSpawner.make((command) =>
487+
Effect.sync(() => {
488+
const args = commandArgs(command);
489+
if (args.includes("-N")) {
490+
forwards += 1;
491+
stale = false;
492+
return makeRunningProcess(() => {
493+
kills += 1;
494+
});
495+
}
496+
if (args.includes("--")) return makeSuccessfulProcess('{"remotePort":3773}\n');
497+
if (args.includes("sh")) stops += 1;
498+
return makeSuccessfulProcess("");
499+
}),
500+
);
501+
const httpClient = HttpClient.make((request) =>
502+
stale
503+
? Deferred.succeed(probeStarted, undefined).pipe(Effect.andThen(Effect.never))
504+
: Effect.succeed(HttpClientResponse.fromWeb(request, new Response("", { status: 200 }))),
505+
);
506+
const layer = Layer.mergeAll(
507+
NodeServices.layer,
508+
Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, spawner),
509+
Layer.succeed(HttpClient.HttpClient, httpClient),
510+
Layer.succeed(NetService.NetService, testNetService),
511+
SshAuth.SshPasswordPrompt.disabledLayer,
512+
SshTunnel.SshEnvironmentManager.layer({ resolveCliRunner: Effect.succeed(ARCHIVE) }),
513+
);
514+
yield* Effect.gen(function* () {
515+
const manager = yield* SshTunnel.SshEnvironmentManager;
516+
const target = { alias: "devbox", hostname: "devbox", username: null, port: null };
517+
yield* manager.ensureEnvironment(target);
518+
stale = true;
519+
const reconnect = yield* Effect.forkChild(manager.ensureEnvironment(target));
520+
yield* Deferred.await(probeStarted);
521+
yield* TestClock.adjust("2 seconds");
522+
yield* Fiber.join(reconnect);
523+
assert.equal(forwards, 2);
524+
assert.equal(kills, 1);
525+
assert.equal(stops, 0);
526+
yield* manager.disconnectEnvironment(target);
527+
assert.equal(stops, 1);
528+
}).pipe(Effect.provide(layer), Effect.scoped);
529+
}),
530+
);
531+
487532
it.effect.each(["successful stop", "failed stop"] as const)(
488533
"closes the tunnel scope and starts fresh after a %s",
489534
(mode) => {
@@ -799,3 +844,91 @@ describe("archive runner script", () => {
799844
60_000,
800845
);
801846
});
847+
848+
describe.skipIf(HostProcess.Platform.defaultValue() === "win32")("remote reconnect script", () => {
849+
it.effect.each([
850+
{ ready: true, runtime: "same" },
851+
{ ready: false, runtime: "same" },
852+
{ ready: true, runtime: "different" },
853+
] as const)("preserves a live managed server: %j", ({ ready, runtime }) =>
854+
Effect.gen(function* () {
855+
const fs = yield* FileSystem.FileSystem;
856+
const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
857+
const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-ssh-reconnect-" });
858+
const state = `${root}/.t3/ssh-launch/test`;
859+
yield* fs.makeDirectory(state, { recursive: true });
860+
yield* fs.makeDirectory(`${root}/.t3/userdata`, { recursive: true });
861+
const server = yield* spawner.spawn(ChildProcess.make("sleep", ["120"]));
862+
const other = yield* spawner.spawn(ChildProcess.make("sleep", ["120"]));
863+
yield* fs.writeFileString(`${state}/pid`, String(server.pid));
864+
yield* fs.writeFileString(`${state}/port`, "3773");
865+
yield* fs.writeFileString(`${state}/managed`, "managed");
866+
yield* fs.writeFileString(`${state}/run-t3.sh`, "old runner");
867+
yield* fs.writeFileString(
868+
`${root}/.t3/userdata/server-runtime.json`,
869+
JSON.stringify({
870+
pid: runtime === "same" ? Number(server.pid) : Number(other.pid),
871+
port: runtime === "same" ? 3773 : 4773,
872+
origin: `http://127.0.0.1:${runtime === "same" ? 3773 : 4773}`,
873+
}),
874+
);
875+
yield* fs.makeDirectory(`${root}/bin`);
876+
yield* fs.writeFileString(
877+
`${root}/bin/node`,
878+
`#!/bin/sh
879+
if [ "$1" = "-" ] && { [ "$#" -eq 4 ] || [ "$2" = ">=20" ]; }; then
880+
cat >/dev/null
881+
if [ "$#" -eq 4 ]; then exit ${ready ? 0 : 1}; fi
882+
exit 0
883+
fi
884+
exec "$REAL_NODE" "$@"
885+
`,
886+
);
887+
yield* fs.chmod(`${root}/bin/node`, 0o700);
888+
const script = `${root}/launch.sh`;
889+
yield* fs.writeFileString(
890+
script,
891+
SshTunnel.buildRemoteLaunchScript({
892+
nodeScriptPath: `${root}/unused.mjs`,
893+
nodeEngineRange: ">=20",
894+
}),
895+
);
896+
const child = yield* spawner.spawn(
897+
ChildProcess.make("sh", [script, "test"], {
898+
env: {
899+
HOME: root,
900+
PATH: `${root}/bin:${process.env.PATH ?? ""}`,
901+
REAL_NODE: process.execPath,
902+
},
903+
extendEnv: false,
904+
}),
905+
);
906+
const [stdout, stderr, code] = yield* Effect.all(
907+
[
908+
child.stdout.pipe(
909+
Stream.decodeText(),
910+
Stream.runFold(
911+
() => "",
912+
(a, b) => a + b,
913+
),
914+
),
915+
child.stderr.pipe(
916+
Stream.decodeText(),
917+
Stream.runFold(
918+
() => "",
919+
(a, b) => a + b,
920+
),
921+
),
922+
child.exitCode,
923+
],
924+
{ concurrency: "unbounded" },
925+
);
926+
assert.isTrue(yield* server.isRunning);
927+
assert.isTrue(yield* other.isRunning);
928+
assert.equal(Number(code), ready ? 0 : 1, stderr);
929+
if (ready) assert.deepEqual(JSON.parse(stdout), { remotePort: 3773, serverKind: "managed" });
930+
else assert.include(stderr, "server was left running");
931+
assert.equal(yield* fs.readFileString(`${state}/pid`), String(server.pid));
932+
}).pipe(Effect.provide(NodeServices.layer)),
933+
);
934+
});

‎packages/ssh/src/tunnel.ts‎

Lines changed: 22 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -559,10 +559,6 @@ trap cleanup_runner_next EXIT
559559
cat >"$RUNNER_NEXT" <<'SH'
560560
@@T3_RUNNER_SCRIPT@@
561561
SH
562-
RUNNER_CHANGED=0
563-
if [ ! -f "$RUNNER_FILE" ] || ! cmp -s "$RUNNER_NEXT" "$RUNNER_FILE"; then
564-
RUNNER_CHANGED=1
565-
fi
566562
mv "$RUNNER_NEXT" "$RUNNER_FILE"
567563
chmod 700 "$RUNNER_FILE"
568564
T3_ARCHIVE_MODE=@@T3_ARCHIVE_MODE@@
@@ -637,53 +633,28 @@ if [ -n "$DEFAULT_RUNTIME_INFO" ]; then
637633
DEFAULT_RUNTIME_PID="\${DEFAULT_RUNTIME_INFO%% *}"
638634
DEFAULT_REMOTE_PORT="\${DEFAULT_RUNTIME_INFO#* }"
639635
fi
640-
if [ -n "$DEFAULT_REMOTE_PORT" ]; then
641-
REMOTE_PORT="$DEFAULT_REMOTE_PORT"
642-
if wait_ready "@@T3_REUSE_READY_TIMEOUT_MS@@"; then
643-
if [ "$REMOTE_MANAGED" = "managed" ]; then
644-
PID_TO_STOP="\${REMOTE_PID:-$DEFAULT_RUNTIME_PID}"
645-
if [ -n "$PID_TO_STOP" ] && kill -0 "$PID_TO_STOP" 2>/dev/null; then
646-
kill "$PID_TO_STOP" 2>/dev/null || true
647-
wait_for_pid_exit "$PID_TO_STOP"
648-
fi
649-
REMOTE_PID=""
650-
REMOTE_PORT="$DEFAULT_REMOTE_PORT"
651-
REMOTE_MANAGED="external"
652-
rm -f "$PID_FILE"
653-
printf '%s\\n' "$REMOTE_PORT" >"$PORT_FILE"
654-
printf 'external\\n' >"$MANAGED_FILE"
655-
else
656-
printf '%s\\n' "$REMOTE_PORT" >"$PORT_FILE"
657-
printf 'external\\n' >"$MANAGED_FILE"
658-
REMOTE_PID=""
659-
REMOTE_MANAGED="external"
660-
fi
661-
else
662-
REMOTE_PID="$(cat "$PID_FILE" 2>/dev/null || true)"
663-
REMOTE_PORT="$(cat "$PORT_FILE" 2>/dev/null || true)"
664-
REMOTE_MANAGED="$(cat "$MANAGED_FILE" 2>/dev/null || true)"
665-
fi
666-
fi
667-
if [ "$REMOTE_MANAGED" = "external" ]; then
668-
if [ -z "$REMOTE_PORT" ] || ! wait_ready "@@T3_REUSE_READY_TIMEOUT_MS@@"; then
669-
REMOTE_PID=""
670-
REMOTE_PORT=""
671-
REMOTE_MANAGED=""
636+
# Reconnect owns the local tunnel, not the lifetime of an already running server.
637+
# Prefer the server we launched, even when the default runtime belongs to another
638+
# instance or this client has downloaded a newer runner.
639+
if [ -n "$REMOTE_PID" ] && [ -n "$REMOTE_PORT" ] && kill -0 "$REMOTE_PID" 2>/dev/null; then
640+
REMOTE_MANAGED="managed"
641+
if ! wait_ready "@@T3_READY_TIMEOUT_MS@@"; then
642+
printf 'Remote T3 server is still running but did not become ready. Retry the connection; the server was left running.\\n' >&2
643+
exit 1
672644
fi
673-
elif [ -n "$REMOTE_PID" ] && [ -n "$REMOTE_PORT" ] && kill -0 "$REMOTE_PID" 2>/dev/null; then
674-
if [ "$RUNNER_CHANGED" -eq 1 ]; then
675-
kill "$REMOTE_PID" 2>/dev/null || true
676-
wait_for_pid_exit "$REMOTE_PID"
677-
REMOTE_PID=""
678-
REMOTE_PORT=""
679-
REMOTE_MANAGED=""
680-
elif ! wait_ready "@@T3_REUSE_READY_TIMEOUT_MS@@"; then
681-
kill "$REMOTE_PID" 2>/dev/null || true
682-
wait_for_pid_exit "$REMOTE_PID"
683-
REMOTE_PID=""
684-
REMOTE_PORT=""
685-
REMOTE_MANAGED=""
645+
elif [ -n "$DEFAULT_REMOTE_PORT" ]; then
646+
REMOTE_PID=""
647+
REMOTE_PORT="$DEFAULT_REMOTE_PORT"
648+
REMOTE_MANAGED="external"
649+
if ! wait_ready "@@T3_READY_TIMEOUT_MS@@"; then
650+
printf 'Remote T3 server is still running but did not become ready. Retry the connection; the server was left running.\\n' >&2
651+
exit 1
686652
fi
653+
rm -f "$PID_FILE"
654+
printf '%s\\n' "$REMOTE_PORT" >"$PORT_FILE"
655+
printf 'external\\n' >"$MANAGED_FILE"
656+
elif [ "$REMOTE_MANAGED" = "external" ] && [ -n "$REMOTE_PORT" ] && wait_ready "@@T3_REUSE_READY_TIMEOUT_MS@@"; then
657+
REMOTE_PID=""
687658
else
688659
REMOTE_PID=""
689660
REMOTE_PORT=""
@@ -1635,6 +1606,8 @@ const makeSshEnvironmentManager = Effect.fn("ssh/tunnel.SshEnvironmentManager.ma
16351606
remotePort: entry.remotePort,
16361607
cause: readinessExit.cause,
16371608
});
1609+
// Closing a stale forward must not run the manager-owned remote stop.
1610+
tunnels.delete(key);
16381611
yield* closeTunnelEntry(entry);
16391612
}
16401613

0 commit comments

Comments
 (0)