Skip to content

Commit 29975cd

Browse files
fix(ssh): launch lock recovery and unpublished servers stay single-owner
A waiter could delete a recovery lock that a live launch had just taken, letting two launches into the critical section. Waiters now never remove another launch's recovery lock; a launch SIGKILLed inside that window leaves one behind, and the bounded wait fails naming the file to delete. A launch that finds no server-runtime.json now checks the other aliases' launch records for a live managed server that answers the reuse probe and adopts it as external, instead of starting a second server that loses the home's owner lock.
1 parent 111bcfe commit 29975cd

2 files changed

Lines changed: 93 additions & 63 deletions

File tree

‎packages/ssh/src/tunnel.test.ts‎

Lines changed: 65 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -98,8 +98,8 @@ const ARCHIVE = { archiveVersion: "1.2.3-preview.20260911.4" } as const;
9898
// Emulates `t3 serve`: it takes the home owner lock at startup, then binds
9999
// once a second server has tried the same home (or after 2s), so a concurrent
100100
// launch always sees the home still unclaimed. Like the real server, it
101-
// publishes server-runtime.json some time after it starts answering HTTP.
102-
const FAKE_T3_SERVE = `const fs = require("node:fs");
101+
// publishes server-runtime.json after it starts answering HTTP, if at all.
102+
const fakeT3Serve = (publishDelayMs: number | null) => `const fs = require("node:fs");
103103
const http = require("node:http");
104104
const path = require("node:path");
105105
const args = process.argv.slice(2);
@@ -122,9 +122,10 @@ const serve = (tries) => {
122122
}
123123
http.createServer((_, response) => response.end()).listen(port, "127.0.0.1", () => {
124124
const origin = "http://127.0.0.1:" + port;
125+
${publishDelayMs === null ? "return;" : ""}
125126
setTimeout(() => {
126127
fs.writeFileSync(path.join(userdata, "server-runtime.json"), JSON.stringify({ pid: process.pid, port, origin }));
127-
}, 1_500);
128+
}, ${publishDelayMs});
128129
});
129130
};
130131
serve(40);
@@ -701,61 +702,71 @@ describe("ssh tunnel scripts", () => {
701702
);
702703

703704
it.live("concurrent launches on one remote home record the server that owns it", () =>
704-
Effect.gen(function* () {
705-
const fs = yield* FileSystem.FileSystem;
706-
const path = yield* Path.Path;
707-
const home = yield* fs.makeTempDirectoryScoped({ prefix: "t3-ssh-launch-" });
708-
yield* Effect.addFinalizer(() =>
709-
fs.readFileString(path.join(home, "servers")).pipe(
710-
Effect.map((pids) => {
711-
for (const pid of pids.split("\n").filter(Boolean)) {
712-
try {
713-
process.kill(Number(pid), "SIGKILL");
714-
} catch {}
715-
}
716-
}),
717-
Effect.ignore,
718-
),
719-
);
720-
const serverScript = path.join(home, "fake-t3.cjs");
721-
yield* fs.writeFileString(serverScript, FAKE_T3_SERVE);
722-
// A launch that died holding the lock must not block or double-admit later ones.
723-
const launchLock = path.join(home, ".t3", "ssh-launch", ".launch.lock");
724-
yield* fs.makeDirectory(path.dirname(launchLock), { recursive: true });
725-
yield* fs.writeFileString(launchLock, "999999999\n");
726-
const targets = [
727-
{ alias: "devbox", hostname: "devbox", username: null, port: null },
728-
{ alias: "devbox-tailnet", hostname: "devbox-tailnet", username: null, port: null },
729-
];
705+
// The second row's server never publishes server-runtime.json.
706+
Effect.forEach([1_500, null], (publishDelayMs) =>
707+
Effect.gen(function* () {
708+
const fs = yield* FileSystem.FileSystem;
709+
const path = yield* Path.Path;
710+
const home = yield* fs.makeTempDirectoryScoped({ prefix: "t3-ssh-launch-" });
711+
const serversFile = path.join(home, "servers");
712+
yield* Effect.addFinalizer(() =>
713+
fs.readFileString(serversFile).pipe(
714+
Effect.map((pids) => {
715+
for (const pid of pids.split("\n").filter(Boolean)) {
716+
try {
717+
process.kill(Number(pid), "SIGKILL");
718+
} catch {}
719+
}
720+
}),
721+
Effect.ignore,
722+
),
723+
);
724+
const serverScript = path.join(home, "fake-t3.cjs");
725+
yield* fs.writeFileString(serverScript, fakeT3Serve(publishDelayMs));
726+
// A launch that died holding the lock must not block or double-admit later ones.
727+
const launchLock = path.join(home, ".t3", "ssh-launch", ".launch.lock");
728+
yield* fs.makeDirectory(path.dirname(launchLock), { recursive: true });
729+
yield* fs.writeFileString(launchLock, "999999999\n");
730+
const targets = [
731+
{ alias: "devbox", hostname: "devbox", username: null, port: null },
732+
{ alias: "devbox-tailnet", hostname: "devbox-tailnet", username: null, port: null },
733+
];
730734

731-
const launches = yield* Effect.forEach(
732-
targets,
733-
(target) =>
734-
SshTunnel.launchOrReuseRemoteServer(target, undefined, { nodeScriptPath: serverScript }),
735-
{ concurrency: "unbounded" },
736-
).pipe(Effect.provide(localRemoteShellLayer(home)));
735+
const launches = yield* Effect.forEach(
736+
targets,
737+
(target) =>
738+
SshTunnel.launchOrReuseRemoteServer(target, undefined, {
739+
nodeScriptPath: serverScript,
740+
}),
741+
{ concurrency: "unbounded" },
742+
).pipe(Effect.provide(localRemoteShellLayer(home)));
737743

738-
const ownerPid = Number(
739-
yield* fs.readFileString(path.join(home, ".t3", "userdata", "server.lock")),
740-
);
741-
assert.include(
742-
launches.map((launch) => launch.remoteServerKind),
743-
"managed",
744-
);
745-
for (const [index, launch] of launches.entries()) {
746-
if (launch.remoteServerKind !== "managed") continue;
747-
const stateKey = yield* remoteStateKey(targets[index]!);
748-
const recordedPid = yield* fs.readFileString(
749-
path.join(home, ".t3", "ssh-launch", stateKey, "pid"),
744+
const ownerPid = Number(
745+
yield* fs.readFileString(path.join(home, ".t3", "userdata", "server.lock")),
750746
);
751-
assert.equal(
752-
Number(recordedPid.trim()),
753-
ownerPid,
754-
`${targets[index]!.alias} reported a managed server, but its recorded PID is not the server that owns the remote home`,
747+
assert.sameMembers(
748+
launches.map((launch) => launch.remoteServerKind),
749+
["managed", "external"],
755750
);
756-
}
757-
assert.isFalse(yield* fs.exists(launchLock));
758-
}).pipe(Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)), Effect.scoped),
751+
assert.equal(launches[0]!.remotePort, launches[1]!.remotePort);
752+
assert.deepEqual((yield* fs.readFileString(serversFile)).trim().split("\n"), [
753+
String(ownerPid),
754+
]);
755+
for (const [index, launch] of launches.entries()) {
756+
if (launch.remoteServerKind !== "managed") continue;
757+
const stateKey = yield* remoteStateKey(targets[index]!);
758+
const recordedPid = yield* fs.readFileString(
759+
path.join(home, ".t3", "ssh-launch", stateKey, "pid"),
760+
);
761+
assert.equal(
762+
Number(recordedPid.trim()),
763+
ownerPid,
764+
`${targets[index]!.alias} reported a managed server, but its recorded PID is not the server that owns the remote home`,
765+
);
766+
}
767+
assert.isFalse(yield* fs.exists(launchLock));
768+
}).pipe(Effect.scoped),
769+
).pipe(Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))),
759770
);
760771
});
761772

‎packages/ssh/src/tunnel.ts‎

Lines changed: 28 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -584,15 +584,11 @@ elif ! ensure_remote_node_path; then
584584
exit 1
585585
fi
586586
# Recovery runs under its own lock and re-reads the owner there, so a waiter
587-
# that judged an earlier lock stale cannot remove its successor's lock.
587+
# that judged an earlier lock stale cannot remove its successor's lock. No
588+
# launch removes another's recovery lock: only SIGKILL inside these lines
589+
# leaves one behind, and the wait below then fails naming it.
588590
discard_stale_launch_lock() {
589-
if ! (set -C; printf '%s\\n' "$$" >"$LAUNCH_BREAK_LOCK") 2>/dev/null; then
590-
LAUNCH_BREAK_OWNER="$(cat "$LAUNCH_BREAK_LOCK" 2>/dev/null || true)"
591-
if [ -z "$LAUNCH_BREAK_OWNER" ] || ! kill -0 "$LAUNCH_BREAK_OWNER" 2>/dev/null; then
592-
rm -f "$LAUNCH_BREAK_LOCK"
593-
fi
594-
return 1
595-
fi
591+
(set -C; printf '%s\\n' "$$" >"$LAUNCH_BREAK_LOCK") 2>/dev/null || return 1
596592
if [ "$(cat "$LAUNCH_LOCK" 2>/dev/null || true)" = "$1" ]; then
597593
rm -f "$LAUNCH_LOCK"
598594
fi
@@ -619,7 +615,11 @@ while ! (set -C; printf '%s\\n' "$$" >"$LAUNCH_LOCK") 2>/dev/null; do
619615
fi
620616
fi
621617
if [ "$LAUNCH_LOCK_WAITED" -ge @@T3_LAUNCH_LOCK_WAIT_SECONDS@@ ]; then
622-
printf 'Another T3 server launch has held %s for too long.\\n' "$LAUNCH_LOCK" >&2
618+
if [ -e "$LAUNCH_BREAK_LOCK" ]; then
619+
printf 'A T3 server launch stopped while recovering %s. If no launch is running, delete %s.\\n' "$LAUNCH_LOCK" "$LAUNCH_BREAK_LOCK" >&2
620+
else
621+
printf 'Another T3 server launch has held %s for too long.\\n' "$LAUNCH_LOCK" >&2
622+
fi
623623
exit 1
624624
fi
625625
sleep 1
@@ -762,6 +762,25 @@ else
762762
REMOTE_PORT=""
763763
REMOTE_MANAGED=""
764764
fi
765+
# The server may answer long before it publishes its runtime record, or never
766+
# publish one; another alias's launch record still names it. Only one server
767+
# owns the home, so the first live one decides.
768+
if [ -z "$REMOTE_PORT" ] && [ -z "$DEFAULT_RUNTIME_INFO" ]; then
769+
for SIBLING_DIR in "$DEFAULT_SERVER_HOME"/ssh-launch/*; do
770+
[ "$SIBLING_DIR" != "$STATE_DIR" ] && [ "$(cat "$SIBLING_DIR/managed" 2>/dev/null || true)" = "managed" ] || continue
771+
server_running "$(cat "$SIBLING_DIR/pid" 2>/dev/null || true)" || continue
772+
REMOTE_PORT="$(cat "$SIBLING_DIR/port" 2>/dev/null || true)"
773+
if [ -n "$REMOTE_PORT" ] && wait_ready "@@T3_REUSE_READY_TIMEOUT_MS@@"; then
774+
REMOTE_MANAGED="external"
775+
rm -f "$PID_FILE"
776+
printf '%s\\n' "$REMOTE_PORT" >"$PORT_FILE"
777+
printf 'external\\n' >"$MANAGED_FILE"
778+
else
779+
REMOTE_PORT=""
780+
fi
781+
break
782+
done
783+
fi
765784
if [ -z "$REMOTE_PORT" ]; then
766785
REMOTE_PORT="$(pick_port)" || true
767786
if [ -z "$REMOTE_PORT" ]; then

0 commit comments

Comments
 (0)