Skip to content

Commit 1a19b2e

Browse files
author
PR Batch Tester
committed
fix(ssh): allow lock waits through cold server installs
1 parent f76e1e5 commit 1a19b2e

2 files changed

Lines changed: 94 additions & 6 deletions

File tree

‎packages/ssh/src/tunnel.test.ts‎

Lines changed: 84 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,52 @@ describe("ssh tunnel scripts", () => {
106106
);
107107
});
108108

109+
it.effect.each([
110+
{ operation: "launch", runner: "archive" },
111+
{ operation: "launch", runner: "node" },
112+
{ operation: "pairing", runner: "archive" },
113+
{ operation: "pairing", runner: "node" },
114+
] as const)("lets $runner $operation finish after a full lock wait", ({ operation, runner }) =>
115+
Effect.gen(function* () {
116+
const started = yield* Deferred.make<void>();
117+
const finish = yield* Deferred.make<void>();
118+
const spawner = ChildProcessSpawner.make(() =>
119+
Effect.gen(function* () {
120+
yield* Deferred.succeed(started, undefined);
121+
const result = operation === "launch" ? { remotePort: 3774 } : { credential: "test" };
122+
return {
123+
...makeSuccessfulProcess(JSON.stringify(result)),
124+
exitCode: Deferred.await(finish).pipe(Effect.as(ChildProcessSpawner.ExitCode(0))),
125+
};
126+
}),
127+
);
128+
const target = { alias: "devbox", hostname: "devbox", username: null, port: null };
129+
const options = runner === "archive" ? ARCHIVE : { nodeScriptPath: "/test/server.js" };
130+
const command = Effect.gen(function* () {
131+
if (operation === "launch") {
132+
const result = yield* SshTunnel.launchOrReuseRemoteServer(target, undefined, options);
133+
assert.deepEqual(result, { remotePort: 3774, remoteServerKind: null });
134+
} else {
135+
const result = yield* SshTunnel.issueRemotePairingToken(target, undefined, options);
136+
assert.deepEqual(result, { credential: "test" });
137+
}
138+
});
139+
const fiber = yield* command.pipe(
140+
Effect.provide(
141+
Layer.merge(
142+
NodeServices.layer,
143+
Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, spawner),
144+
),
145+
),
146+
Effect.forkScoped,
147+
);
148+
yield* Deferred.await(started);
149+
yield* TestClock.adjust(Duration.millis(900_001));
150+
yield* Deferred.succeed(finish, undefined);
151+
yield* Fiber.join(fiber);
152+
}).pipe(Effect.scoped),
153+
);
154+
109155
it.effect("accepts launch JSON after remote shell startup noise", () => {
110156
const target = {
111157
alias: "devbox",
@@ -371,7 +417,7 @@ describe("ssh tunnel scripts", () => {
371417
// bind unless both sockets set SO_REUSEPORT, so on Linux the preload adds it to
372418
// the picker's probe and the holder sets it too.
373419

374-
const LOCK_WAIT_SITE = ' if [ "$LOCK_WAIT_COUNT" -ge 200 ]; then';
420+
const LOCK_WAIT_SITE = ' if [ "$LOCK_WAIT_COUNT" -ge 9000 ]; then';
375421
// Gives up after the first failed claim and recovery attempt instead of waiting.
376422
const FAIL_FAST: readonly [string, string] = [
377423
LOCK_WAIT_SITE,
@@ -683,6 +729,43 @@ server.listen(Number(flag("--port")), "127.0.0.1", () => {
683729
}).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
684730
);
685731

732+
it.live("waits past the former 20-second limit for a live owner to finish", () =>
733+
Effect.gen(function* () {
734+
const { launch, writeVariant } = yield* prepareLaunch;
735+
const holding = yield* Deferred.make<void>();
736+
const releaseHolder = yield* Deferred.make<void>();
737+
const holder = yield* writeVariant("slow-holder", [HOLD_LOCK]);
738+
const contender = yield* writeVariant("patient-contender", [
739+
["LOCK_WAIT_COUNT=0", "LOCK_WAIT_COUNT=200"],
740+
[" sleep 0.1", ' printf "waiting\\n"; read -r release'],
741+
]);
742+
const first = yield* launch(
743+
"first-alias",
744+
{
745+
onLine: (line) =>
746+
line === "holding"
747+
? Deferred.succeed(holding, undefined).pipe(Effect.asVoid)
748+
: Effect.void,
749+
release: Deferred.await(releaseHolder),
750+
},
751+
holder,
752+
).pipe(Effect.forkScoped);
753+
yield* Deferred.await(holding);
754+
const second = yield* launch(
755+
"second-alias",
756+
{
757+
onLine: (line) =>
758+
line === "waiting"
759+
? Deferred.succeed(releaseHolder, undefined).pipe(Effect.asVoid)
760+
: Effect.void,
761+
release: Fiber.join(first).pipe(Effect.orDie, Effect.asVoid),
762+
},
763+
contender,
764+
);
765+
assert.deepEqual(second, yield* Fiber.join(first));
766+
}).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
767+
);
768+
686769
it.live("does not let a late recovery remove the next owner's launch lock", () =>
687770
Effect.gen(function* () {
688771
const { fs, run, launch, writeVariant, lockPath, exitedPid } = yield* prepareLaunch;

‎packages/ssh/src/tunnel.ts‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,8 @@ const REMOTE_ARCHIVE_CHECKSUMS_SECONDS = 30;
5858
const REMOTE_ARCHIVE_DOWNLOAD_SECONDS = 240;
5959
const REMOTE_ARCHIVE_LOCK_WAIT_SECONDS = 360;
6060
const REMOTE_ARCHIVE_LAUNCH_TIMEOUT_MS = 900_000;
61+
// Any state operation may queue behind a cold archive launch, then needs its own work budget.
62+
const REMOTE_STATE_LOCK_WAIT_MS = REMOTE_ARCHIVE_LAUNCH_TIMEOUT_MS;
6163
const REMOTE_REUSE_READY_TIMEOUT_MS = 2_000;
6264

6365
export interface RemoteT3RunnerOptions {
@@ -623,7 +625,7 @@ while ! ln "$LAUNCH_LOCK_RECORD" "$LAUNCH_LOCK" 2>/dev/null; do
623625
if reclaim_launch_lock; then
624626
continue
625627
fi
626-
if [ "$LOCK_WAIT_COUNT" -ge 200 ]; then
628+
if [ "$LOCK_WAIT_COUNT" -ge ${REMOTE_STATE_LOCK_WAIT_MS / 100} ]; then
627629
printf 'Another SSH operation holds %s. Retry after it finishes.\\n' "$LAUNCH_LOCK" >&2
628630
exit 1
629631
fi
@@ -1061,9 +1063,9 @@ export const launchOrReuseRemoteServer = Effect.fn("ssh/tunnel.launchOrReuseRemo
10611063
const result = yield* runSshCommand(target, {
10621064
remoteCommandArgs: ["sh", "-l", "-s", "--", stateKey],
10631065
stdin: buildRemoteLaunchScript(runner),
1064-
timeoutMs: isNodeScriptRunner(runner)
1065-
? REMOTE_LAUNCH_TIMEOUT_MS
1066-
: REMOTE_ARCHIVE_LAUNCH_TIMEOUT_MS,
1066+
timeoutMs:
1067+
REMOTE_STATE_LOCK_WAIT_MS +
1068+
(isNodeScriptRunner(runner) ? REMOTE_LAUNCH_TIMEOUT_MS : REMOTE_ARCHIVE_LAUNCH_TIMEOUT_MS),
10671069
...(input?.authSecret === undefined ? {} : { authSecret: input.authSecret }),
10681070
...(input?.batchMode === undefined ? {} : { batchMode: input.batchMode }),
10691071
...(input?.interactiveAuth === undefined ? {} : { interactiveAuth: input.interactiveAuth }),
@@ -1122,7 +1124,9 @@ export const issueRemotePairingToken = Effect.fn("ssh/tunnel.issueRemotePairingT
11221124
const result = yield* runSshCommand(target, {
11231125
remoteCommandArgs: ["sh", "-s"],
11241126
stdin: buildRemotePairingScript(stateKey, runner),
1125-
...(isNodeScriptRunner(runner) ? {} : { timeoutMs: REMOTE_ARCHIVE_LAUNCH_TIMEOUT_MS }),
1127+
timeoutMs:
1128+
REMOTE_STATE_LOCK_WAIT_MS +
1129+
(isNodeScriptRunner(runner) ? REMOTE_LAUNCH_TIMEOUT_MS : REMOTE_ARCHIVE_LAUNCH_TIMEOUT_MS),
11261130
...(input?.authSecret === undefined ? {} : { authSecret: input.authSecret }),
11271131
...(input?.batchMode === undefined ? {} : { batchMode: input.batchMode }),
11281132
...(input?.interactiveAuth === undefined ? {} : { interactiveAuth: input.interactiveAuth }),
@@ -1174,6 +1178,7 @@ const stopRemoteServer = Effect.fn("ssh/tunnel.stopRemoteServer")(function* (
11741178
yield* runSshCommand(target, {
11751179
remoteCommandArgs: ["sh", "-s"],
11761180
stdin: buildRemoteStopScript(stateKey),
1181+
timeoutMs: REMOTE_STATE_LOCK_WAIT_MS + REMOTE_LAUNCH_TIMEOUT_MS,
11771182
...(input?.authSecret === undefined ? {} : { authSecret: input.authSecret }),
11781183
...(input?.batchMode === undefined ? {} : { batchMode: input.batchMode }),
11791184
...(input?.interactiveAuth === undefined ? {} : { interactiveAuth: input.interactiveAuth }),

0 commit comments

Comments
 (0)