Skip to content

Commit 20c3dd3

Browse files
fix(ssh): concurrent launches on one remote home share one server (#32587)
Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent 9b352a0 commit 20c3dd3

2 files changed

Lines changed: 250 additions & 13 deletions

File tree

‎packages/ssh/src/tunnel.test.ts‎

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,9 @@ import * as Deferred from "effect/Deferred";
1313
import * as Duration from "effect/Duration";
1414
import * as Effect from "effect/Effect";
1515
import * as Fiber from "effect/Fiber";
16+
import * as FileSystem from "effect/FileSystem";
1617
import * as Layer from "effect/Layer";
18+
import * as Path from "effect/Path";
1719
import * as PlatformError from "effect/PlatformError";
1820
import * as Schema from "effect/Schema";
1921
import * as Result from "effect/Result";
@@ -24,6 +26,7 @@ import { HttpClient, HttpClientError, HttpClientResponse } from "effect/http";
2426
import { ChildProcess, ChildProcessSpawner } from "effect/process";
2527

2628
import * as SshAuth from "./auth.ts";
29+
import { remoteStateKey } from "./command.ts";
2730
import { SshCommandError, SshPasswordPromptError, SshReadinessError } from "./errors.ts";
2831
import * as SshTunnel from "./tunnel.ts";
2932

@@ -92,6 +95,73 @@ function commandArgs(command: ChildProcess.Command): ReadonlyArray<string> {
9295

9396
const ARCHIVE = { archiveVersion: "1.2.3-preview.20260911.4" } as const;
9497

98+
// Emulates `t3 serve`: it takes the home owner lock at startup, then binds
99+
// once a second server has tried the same home (or after 2s), so a concurrent
100+
// launch always sees the home still unclaimed. Like the real server, it
101+
// publishes server-runtime.json after it starts answering HTTP, if at all.
102+
const fakeT3Serve = (publishDelayMs: number | null) => `const fs = require("node:fs");
103+
const http = require("node:http");
104+
const path = require("node:path");
105+
const args = process.argv.slice(2);
106+
const port = Number(args[args.indexOf("--port") + 1]);
107+
const userdata = path.join(args[args.indexOf("--base-dir") + 1], "userdata");
108+
const servers = path.join(process.env.HOME, "servers");
109+
fs.mkdirSync(userdata, { recursive: true });
110+
fs.appendFileSync(servers, process.pid + "\\n");
111+
try {
112+
fs.writeFileSync(path.join(userdata, "server.lock"), String(process.pid), { flag: "wx" });
113+
} catch {
114+
console.error("ServerAlreadyRunningError: another server owns " + userdata);
115+
process.exit(1);
116+
}
117+
setTimeout(() => process.exit(0), 60_000);
118+
const serve = (tries) => {
119+
if (tries > 0 && fs.readFileSync(servers, "utf8").trim().split("\\n").length < 2) {
120+
setTimeout(() => serve(tries - 1), 50);
121+
return;
122+
}
123+
http.createServer((_, response) => response.end()).listen(port, "127.0.0.1", () => {
124+
const origin = "http://127.0.0.1:" + port;
125+
${publishDelayMs === null ? "return;" : ""}
126+
setTimeout(() => {
127+
fs.writeFileSync(path.join(userdata, "server-runtime.json"), JSON.stringify({ pid: process.pid, port, origin }));
128+
}, ${publishDelayMs});
129+
});
130+
};
131+
serve(40);
132+
`;
133+
134+
// Runs the remote half of an SSH command (`sh -l -s -- <key>`) as a local shell
135+
// whose HOME is the given directory. Every launch after the first starts once
136+
// a server is running (or after 10s), which lines up the race deterministically.
137+
const localRemoteShellLayer = (home: string) =>
138+
Layer.effect(
139+
ChildProcessSpawner.ChildProcessSpawner,
140+
Effect.gen(function* () {
141+
const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
142+
let launches = 0;
143+
return ChildProcessSpawner.make((command) => {
144+
if (command._tag !== "StandardCommand") return spawner.spawn(command);
145+
const gate =
146+
launches++ === 0
147+
? ""
148+
: 'i=0; while [ ! -s "$HOME/servers" ] && [ "$i" -lt 200 ]; do sleep 0.05; i=$((i+1)); done; ';
149+
return spawner.spawn(
150+
ChildProcess.make(
151+
"sh",
152+
["-c", `${gate}exec sh -s -- "$1"`, "sh", ...commandArgs(command).slice(-1)],
153+
{
154+
...command.options,
155+
env: { ...command.options.env, HOME: home },
156+
// Like an SSH session ending, the shell's exit leaves its nohup'd server running.
157+
detached: false,
158+
},
159+
),
160+
);
161+
});
162+
}),
163+
);
164+
95165
describe("ssh tunnel scripts", () => {
96166
it("rejects archive versions that are not a single exact version segment", () => {
97167
for (const archiveVersion of [
@@ -630,6 +700,74 @@ describe("ssh tunnel scripts", () => {
630700
);
631701
}),
632702
);
703+
704+
it.live("concurrent launches on one remote home record the server that owns it", () =>
705+
// The second row's server never publishes server-runtime.json.
706+
Effect.forEach([1_500, null], (publishDelayMs) =>
707+
Effect.gen(function* () {
708+
const fs = yield* FileSystem.FileSystem;
709+
const path = yield* Path.Path;
710+
const home = yield* fs.makeTempDirectoryScoped({ prefix: "t3-ssh-launch-" });
711+
const serversFile = path.join(home, "servers");
712+
yield* Effect.addFinalizer(() =>
713+
fs.readFileString(serversFile).pipe(
714+
Effect.map((pids) => {
715+
for (const pid of pids.split("\n").filter(Boolean)) {
716+
try {
717+
process.kill(Number(pid), "SIGKILL");
718+
} catch {}
719+
}
720+
}),
721+
Effect.ignore,
722+
),
723+
);
724+
const serverScript = path.join(home, "fake-t3.cjs");
725+
yield* fs.writeFileString(serverScript, fakeT3Serve(publishDelayMs));
726+
// A launch that died holding the lock must not block or double-admit later ones.
727+
const launchLock = path.join(home, ".t3", "ssh-launch", ".launch.lock");
728+
yield* fs.makeDirectory(path.dirname(launchLock), { recursive: true });
729+
yield* fs.writeFileString(launchLock, "999999999\n");
730+
const targets = [
731+
{ alias: "devbox", hostname: "devbox", username: null, port: null },
732+
{ alias: "devbox-tailnet", hostname: "devbox-tailnet", username: null, port: null },
733+
];
734+
735+
const launches = yield* Effect.forEach(
736+
targets,
737+
(target) =>
738+
SshTunnel.launchOrReuseRemoteServer(target, undefined, {
739+
nodeScriptPath: serverScript,
740+
}),
741+
{ concurrency: "unbounded" },
742+
).pipe(Effect.provide(localRemoteShellLayer(home)));
743+
744+
const ownerPid = Number(
745+
yield* fs.readFileString(path.join(home, ".t3", "userdata", "server.lock")),
746+
);
747+
assert.sameMembers(
748+
launches.map((launch) => launch.remoteServerKind),
749+
["managed", "external"],
750+
);
751+
assert.equal(launches[0]!.remotePort, launches[1]!.remotePort);
752+
assert.deepEqual((yield* fs.readFileString(serversFile)).trim().split("\n"), [
753+
String(ownerPid),
754+
]);
755+
for (const [index, launch] of launches.entries()) {
756+
if (launch.remoteServerKind !== "managed") continue;
757+
const stateKey = yield* remoteStateKey(targets[index]!);
758+
const recordedPid = yield* fs.readFileString(
759+
path.join(home, ".t3", "ssh-launch", stateKey, "pid"),
760+
);
761+
assert.equal(
762+
Number(recordedPid.trim()),
763+
ownerPid,
764+
`${targets[index]!.alias} reported a managed server, but its recorded PID is not the server that owns the remote home`,
765+
);
766+
}
767+
assert.isFalse(yield* fs.exists(launchLock));
768+
}).pipe(Effect.scoped),
769+
).pipe(Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))),
770+
);
633771
});
634772

635773
// On macOS, Node can bind 127.0.0.1:<port> while another process (such as the

‎packages/ssh/src/tunnel.ts‎

Lines changed: 112 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -53,9 +53,9 @@ const SSH_READY_PROBE_TIMEOUT_MS = 1_000;
5353
const TUNNEL_SHUTDOWN_TIMEOUT_MS = 2_000;
5454
const TUNNEL_STDERR_TAIL_CHARS = 16_384;
5555
const REMOTE_READY_TIMEOUT_MS = 60_000;
56-
const REMOTE_LAUNCH_TIMEOUT_MS = 90_000;
57-
// Matches the node-script launch command's whole ssh run against the same host.
58-
const SSH_TUNNEL_CONNECT_TIMEOUT_MS = REMOTE_LAUNCH_TIMEOUT_MS;
56+
const REMOTE_LAUNCH_BASE_TIMEOUT_MS = 90_000;
57+
// Matches the node-script launch command's ssh run without the launch lock wait.
58+
const SSH_TUNNEL_CONNECT_TIMEOUT_MS = REMOTE_LAUNCH_BASE_TIMEOUT_MS;
5959
const SSH_TUNNEL_FORWARD_PROBE_INTERVAL_MS = 100;
6060
// Cold archive launches download and extract ~70 MB and may wait on an install lock.
6161
// Lock waits must outlast both downloads and extraction; SSH must outlast install or wait plus readiness.
@@ -65,6 +65,23 @@ const REMOTE_ARCHIVE_DOWNLOAD_SECONDS = 240;
6565
const REMOTE_ARCHIVE_LOCK_WAIT_SECONDS = 360;
6666
const REMOTE_ARCHIVE_LAUNCH_TIMEOUT_MS = 900_000;
6767
const REMOTE_REUSE_READY_TIMEOUT_MS = 2_000;
68+
const REMOTE_PID_EXIT_WAIT_MS = 2_000;
69+
// The server answers HTTP before it writes server-runtime.json, which is how
70+
// a launch for another alias finds it.
71+
const REMOTE_RUNTIME_RECORD_WAIT_SECONDS = 10;
72+
// A launch holds the home lock through up to two reuse probes, readiness, two
73+
// server exit waits, and the runtime record wait; each probe may overrun its
74+
// deadline by one request.
75+
const REMOTE_LAUNCH_LOCK_WAIT_SECONDS = Math.ceil(
76+
(REMOTE_READY_TIMEOUT_MS +
77+
2 * REMOTE_REUSE_READY_TIMEOUT_MS +
78+
2 * REMOTE_PID_EXIT_WAIT_MS +
79+
REMOTE_RUNTIME_RECORD_WAIT_SECONDS * 1000 +
80+
4 * SSH_READY_PROBE_TIMEOUT_MS) /
81+
1000,
82+
);
83+
const REMOTE_LAUNCH_TIMEOUT_MS =
84+
REMOTE_LAUNCH_LOCK_WAIT_SECONDS * 1000 + REMOTE_LAUNCH_BASE_TIMEOUT_MS;
6885

6986
export interface RemoteT3RunnerOptions {
7087
readonly nodeScriptPath?: string | null;
@@ -537,30 +554,82 @@ MANAGED_FILE="$STATE_DIR/managed"
537554
LOG_FILE="$STATE_DIR/server.log"
538555
RUNNER_FILE="$STATE_DIR/run-t3.sh"
539556
RUNNER_NEXT="$STATE_DIR/run-t3.next.$$"
557+
LAUNCH_LOCK="$DEFAULT_SERVER_HOME/ssh-launch/.launch.lock"
558+
LAUNCH_BREAK_LOCK="$LAUNCH_LOCK.break"
540559
mkdir -p "$STATE_DIR"
541-
cleanup_runner_next() {
560+
cleanup_launch() {
542561
rm -f "$RUNNER_NEXT"
562+
for OWNED_LOCK in "$LAUNCH_LOCK" "$LAUNCH_BREAK_LOCK"; do
563+
if [ "$(cat "$OWNED_LOCK" 2>/dev/null || true)" = "$$" ]; then
564+
rm -f "$OWNED_LOCK"
565+
fi
566+
done
543567
}
544-
trap cleanup_runner_next EXIT
568+
trap cleanup_launch EXIT
569+
# A dropped SSH session signals the launch (PIPE on its next write); exit so
570+
# the lock is released.
571+
trap 'exit 1' HUP INT PIPE TERM
545572
cat >"$RUNNER_NEXT" <<'SH'
546573
@@T3_RUNNER_SCRIPT@@
547574
SH
548-
RUNNER_CHANGED=0
549-
if [ ! -f "$RUNNER_FILE" ] || ! cmp -s "$RUNNER_NEXT" "$RUNNER_FILE"; then
550-
RUNNER_CHANGED=1
551-
fi
552-
mv "$RUNNER_NEXT" "$RUNNER_FILE"
553-
chmod 700 "$RUNNER_FILE"
575+
chmod 700 "$RUNNER_NEXT"
554576
T3_ARCHIVE_MODE=@@T3_ARCHIVE_MODE@@
555577
if [ "$T3_ARCHIVE_MODE" = "1" ]; then
556578
# The archive ships the helpers below inside the executable; the remote
557579
# needs no Node at all. Resolving the runner once here also downloads the
558580
# archive before the port and readiness probes rely on it.
559-
"$RUNNER_FILE" --version >/dev/null
581+
"$RUNNER_NEXT" --version >/dev/null
560582
elif ! ensure_remote_node_path; then
561583
printf 'Remote host is missing node on PATH. Install Node or configure a supported version manager for non-interactive shells.\\n' >&2
562584
exit 1
563585
fi
586+
# Recovery runs under its own lock and re-reads the owner there, so a waiter
587+
# that judged an earlier lock stale cannot remove its successor's lock. No
588+
# launch removes another's recovery lock: only SIGKILL inside these lines
589+
# leaves one behind, and the wait below then fails naming it.
590+
discard_stale_launch_lock() {
591+
(set -C; printf '%s\\n' "$$" >"$LAUNCH_BREAK_LOCK") 2>/dev/null || return 1
592+
if [ "$(cat "$LAUNCH_LOCK" 2>/dev/null || true)" = "$1" ]; then
593+
rm -f "$LAUNCH_LOCK"
594+
fi
595+
rm -f "$LAUNCH_BREAK_LOCK"
596+
}
597+
# Concurrent launches share this home's owner lock and may share this record.
598+
# Hold the lock from comparing the runner until the server is recorded and
599+
# ready, so a waiting launch reuses that server instead of starting or
600+
# replacing one.
601+
LAUNCH_LOCK_WAITED=0
602+
LAUNCH_LOCK_UNOWNED=0
603+
while ! (set -C; printf '%s\\n' "$$" >"$LAUNCH_LOCK") 2>/dev/null; do
604+
LAUNCH_LOCK_OWNER="$(cat "$LAUNCH_LOCK" 2>/dev/null || true)"
605+
if [ -n "$LAUNCH_LOCK_OWNER" ]; then
606+
LAUNCH_LOCK_UNOWNED=0
607+
if ! kill -0 "$LAUNCH_LOCK_OWNER" 2>/dev/null && discard_stale_launch_lock "$LAUNCH_LOCK_OWNER"; then
608+
continue
609+
fi
610+
else
611+
LAUNCH_LOCK_UNOWNED=$((LAUNCH_LOCK_UNOWNED + 1))
612+
if [ "$LAUNCH_LOCK_UNOWNED" -ge 5 ] && discard_stale_launch_lock ""; then
613+
LAUNCH_LOCK_UNOWNED=0
614+
continue
615+
fi
616+
fi
617+
if [ "$LAUNCH_LOCK_WAITED" -ge @@T3_LAUNCH_LOCK_WAIT_SECONDS@@ ]; then
618+
if [ -e "$LAUNCH_BREAK_LOCK" ]; then
619+
printf 'A T3 server launch stopped while recovering %s. If no launch is running, delete %s.\\n' "$LAUNCH_LOCK" "$LAUNCH_BREAK_LOCK" >&2
620+
else
621+
printf 'Another T3 server launch has held %s for too long.\\n' "$LAUNCH_LOCK" >&2
622+
fi
623+
exit 1
624+
fi
625+
sleep 1
626+
LAUNCH_LOCK_WAITED=$((LAUNCH_LOCK_WAITED + 1))
627+
done
628+
RUNNER_CHANGED=0
629+
if [ ! -f "$RUNNER_FILE" ] || ! cmp -s "$RUNNER_NEXT" "$RUNNER_FILE"; then
630+
RUNNER_CHANGED=1
631+
fi
632+
mv "$RUNNER_NEXT" "$RUNNER_FILE"
564633
pick_port() {
565634
if [ "$T3_ARCHIVE_MODE" = "1" ]; then
566635
"$RUNNER_FILE" __ssh-helper pick-port "$PORT_FILE" "@@T3_DEFAULT_REMOTE_PORT@@" "@@T3_REMOTE_PORT_SCAN_WINDOW@@"
@@ -582,7 +651,7 @@ NODE
582651
wait_for_pid_exit() {
583652
PID_TO_WAIT="$1"
584653
WAIT_COUNT=0
585-
while kill -0 "$PID_TO_WAIT" 2>/dev/null && [ "$WAIT_COUNT" -lt 20 ]; do
654+
while kill -0 "$PID_TO_WAIT" 2>/dev/null && [ "$WAIT_COUNT" -lt @@T3_PID_EXIT_WAIT_TICKS@@ ]; do
586655
WAIT_COUNT=$((WAIT_COUNT + 1))
587656
sleep 0.1
588657
done
@@ -693,6 +762,25 @@ else
693762
REMOTE_PORT=""
694763
REMOTE_MANAGED=""
695764
fi
765+
# The server may answer long before it publishes its runtime record, or never
766+
# publish one; another alias's launch record still names it. Only one server
767+
# owns the home, so the first live one decides.
768+
if [ -z "$REMOTE_PORT" ] && [ -z "$DEFAULT_RUNTIME_INFO" ]; then
769+
for SIBLING_DIR in "$DEFAULT_SERVER_HOME"/ssh-launch/*; do
770+
[ "$SIBLING_DIR" != "$STATE_DIR" ] && [ "$(cat "$SIBLING_DIR/managed" 2>/dev/null || true)" = "managed" ] || continue
771+
server_running "$(cat "$SIBLING_DIR/pid" 2>/dev/null || true)" || continue
772+
REMOTE_PORT="$(cat "$SIBLING_DIR/port" 2>/dev/null || true)"
773+
if [ -n "$REMOTE_PORT" ] && wait_ready "@@T3_REUSE_READY_TIMEOUT_MS@@"; then
774+
REMOTE_MANAGED="external"
775+
rm -f "$PID_FILE"
776+
printf '%s\\n' "$REMOTE_PORT" >"$PORT_FILE"
777+
printf 'external\\n' >"$MANAGED_FILE"
778+
else
779+
REMOTE_PORT=""
780+
fi
781+
break
782+
done
783+
fi
696784
if [ -z "$REMOTE_PORT" ]; then
697785
REMOTE_PORT="$(pick_port)" || true
698786
if [ -z "$REMOTE_PORT" ]; then
@@ -720,6 +808,14 @@ if [ -z "$REMOTE_PORT" ]; then
720808
rm -f "$PID_FILE" "$PORT_FILE" "$MANAGED_FILE"
721809
exit 1
722810
fi
811+
# Keep the lock until another alias's launch can find this server, or it
812+
# would start a second one that loses the home's owner lock.
813+
RECORD_DEADLINE=$(($(date +%s) + @@T3_RUNTIME_RECORD_WAIT_SECONDS@@))
814+
while [ "$(date +%s)" -lt "$RECORD_DEADLINE" ]; do
815+
DEFAULT_RUNTIME_INFO="$(resolve_default_runtime_port 2>/dev/null || true)"
816+
[ "\${DEFAULT_RUNTIME_INFO%% *}" = "$REMOTE_PID" ] && break
817+
sleep 0.1
818+
done
723819
fi
724820
printf '{"remotePort":%s,"serverKind":"%s"}\\n' "$REMOTE_PORT" "\${REMOTE_MANAGED:-managed}"
725821
`;
@@ -839,6 +935,9 @@ function buildRemoteLaunchScript(input?: RemoteT3RunnerOptions): string {
839935
T3_READY_TIMEOUT_MS: String(REMOTE_READY_TIMEOUT_MS),
840936
T3_REUSE_READY_TIMEOUT_MS: String(REMOTE_REUSE_READY_TIMEOUT_MS),
841937
T3_READY_PROBE_TIMEOUT_MS: String(SSH_READY_PROBE_TIMEOUT_MS),
938+
T3_PID_EXIT_WAIT_TICKS: String(REMOTE_PID_EXIT_WAIT_MS / 100),
939+
T3_LAUNCH_LOCK_WAIT_SECONDS: String(REMOTE_LAUNCH_LOCK_WAIT_SECONDS),
940+
T3_RUNTIME_RECORD_WAIT_SECONDS: String(REMOTE_RUNTIME_RECORD_WAIT_SECONDS),
842941
});
843942
}
844943

0 commit comments

Comments
 (0)