Skip to content

Commit 6c7d23d

Browse files
fix(client-runtime): slow SSH runtime downloads report their own error, not a setup timeout (pingdotgg#29264)
1 parent c3e5bfc commit 6c7d23d

4 files changed

Lines changed: 226 additions & 7 deletions

File tree

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
import { EnvironmentId } from "@t3tools/contracts";
2+
import { describe, expect, it } from "@effect/vitest";
3+
import * as Effect from "effect/Effect";
4+
import * as Layer from "effect/Layer";
5+
import * as Option from "effect/Option";
6+
import * as Ref from "effect/Ref";
7+
import * as FetchHttpClient from "effect/http/FetchHttpClient";
8+
9+
import * as RemoteEnvironmentAuthorization from "../authorization/service.ts";
10+
import * as RpcSession from "../rpc/session.ts";
11+
import type { ConnectionCatalogEntry } from "./catalog.ts";
12+
import * as ConnectionDriver from "./driver.ts";
13+
import { ConnectionTransientError, SshConnectionTarget, type PreparedConnection } from "./model.ts";
14+
import * as ConnectionResolver from "./resolver.ts";
15+
16+
const sshTarget = (connectionId: string) =>
17+
new SshConnectionTarget({
18+
environmentId: EnvironmentId.make("environment-1"),
19+
label: "Test environment",
20+
connectionId,
21+
});
22+
23+
const ENTRY: ConnectionCatalogEntry = {
24+
target: sshTarget("ssh-first"),
25+
profile: Option.none(),
26+
enabled: true,
27+
alternateRoutes: [{ target: sshTarget("ssh-second"), profile: Option.none() }],
28+
};
29+
30+
describe("ConnectionDriver", () => {
31+
it.effect("reports preparing before each route it prepares", () =>
32+
Effect.gen(function* () {
33+
const stages = yield* Ref.make<ReadonlyArray<string>>([]);
34+
const driver = yield* ConnectionDriver.make.pipe(
35+
Effect.provide(
36+
Layer.mergeAll(
37+
Layer.mock(ConnectionResolver.ConnectionResolver)({
38+
prepare: (entry) =>
39+
Effect.succeed({
40+
environmentId: entry.target.environmentId,
41+
label: entry.target.label,
42+
httpBaseUrl: "http://127.0.0.1:1",
43+
socketUrl: "ws://127.0.0.1:1/ws",
44+
httpAuthorization: null,
45+
target: entry.target,
46+
} satisfies PreparedConnection),
47+
}),
48+
Layer.mock(RpcSession.RpcSessionFactory)({
49+
connect: () =>
50+
Effect.fail(
51+
new ConnectionTransientError({ reason: "transport", detail: "Socket closed." }),
52+
),
53+
}),
54+
Layer.mock(RemoteEnvironmentAuthorization.RemoteEnvironmentAuthorization)({}),
55+
// SSH routes have no HTTP check, so the driver never sends a request.
56+
FetchHttpClient.layer,
57+
),
58+
),
59+
);
60+
61+
yield* driver
62+
.connect(ENTRY, (progress) => Ref.update(stages, (all) => [...all, progress.stage]))
63+
.pipe(Effect.scoped, Effect.flip);
64+
65+
expect(yield* Ref.get(stages)).toEqual([
66+
"preparing",
67+
"preparing",
68+
"opening",
69+
"preparing",
70+
"opening",
71+
]);
72+
}),
73+
);
74+
});

‎packages/client-runtime/src/connection/driver.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -199,6 +199,7 @@ export const make = Effect.gen(function* () {
199199
entry,
200200
(route) => checkRoute(entry, route),
201201
Effect.fnUntraced(function* (route) {
202+
yield* reportProgress({ stage: "preparing" });
202203
const prepared = yield* resolver.prepare(routeEntry(entry, route));
203204
yield* reportProgress({ stage: "opening", prepared });
204205
const session = yield* sessions.connect(prepared);

‎packages/client-runtime/src/connection/supervisor.test.ts‎

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import {
77
import { describe, expect, it } from "@effect/vitest";
88
import * as Deferred from "effect/Deferred";
99
import * as Effect from "effect/Effect";
10+
import * as Fiber from "effect/Fiber";
1011
import * as Layer from "effect/Layer";
1112
import * as Option from "effect/Option";
1213
import * as Random from "effect/Random";
@@ -32,6 +33,7 @@ import {
3233
ConnectionTransientError,
3334
PrimaryConnectionTarget,
3435
RelayConnectionTarget,
36+
SshConnectionTarget,
3537
type ConnectionAttemptError,
3638
type ConnectionTarget,
3739
type NetworkStatus,
@@ -103,6 +105,16 @@ const LAN_THEN_RELAY_ENTRY: ConnectionCatalogEntry = {
103105
enabled: true,
104106
};
105107

108+
const SSH_ENTRY: ConnectionCatalogEntry = {
109+
target: new SshConnectionTarget({
110+
environmentId: TARGET.environmentId,
111+
label: TARGET.label,
112+
connectionId: "ssh-connection",
113+
}),
114+
profile: Option.none(),
115+
enabled: true,
116+
};
117+
106118
const LAN_ENTRY: ConnectionCatalogEntry = {
107119
target: LAN_ROUTE.target,
108120
profile: LAN_ROUTE.profile,
@@ -194,6 +206,7 @@ const makeHarness = Effect.fn("TestConnectionHarness.make")(function* (options?:
194206
target: ConnectionTarget,
195207
reportProgress: (progress: ConnectionDriver.ConnectionDriverProgress) => Effect.Effect<void>,
196208
) {
209+
yield* reportProgress({ stage: "preparing" });
197210
const prepared = yield* prepare(target);
198211
yield* reportProgress({ stage: "opening", prepared });
199212

@@ -556,6 +569,126 @@ describe("EnvironmentSupervisor", () => {
556569
}).pipe(Effect.provide(TestClock.layer())),
557570
);
558571

572+
it.effect("reports a slow SSH preparation failure instead of the setup timeout", () =>
573+
Effect.gen(function* () {
574+
const preparationFailure = new ConnectionTransientError({
575+
reason: "remote-unavailable",
576+
detail:
577+
"Could not prepare the SSH environment: curl: (28) Operation timed out after 240003 milliseconds",
578+
});
579+
const harness = yield* makeHarness({
580+
prepare: () =>
581+
Effect.sleep("4 minutes").pipe(Effect.andThen(Effect.fail(preparationFailure))),
582+
});
583+
const supervisor = yield* EnvironmentSupervisor.make(SSH_ENTRY, {
584+
initiallyDesired: true,
585+
}).pipe(Effect.provide(harness.dependencies));
586+
587+
yield* awaitState(
588+
supervisor.state,
589+
(state) => state.phase === "connecting" && state.stage === "preparing",
590+
);
591+
const firstFailure = yield* awaitState(
592+
supervisor.state,
593+
(state) => state.phase === "backoff",
594+
).pipe(Effect.forkChild({ startImmediately: true }));
595+
yield* TestClock.adjust("4 minutes");
596+
597+
expect(yield* Fiber.join(firstFailure)).toMatchObject({
598+
attempt: 1,
599+
lastFailure: {
600+
_tag: "ConnectionTransientError",
601+
reason: "remote-unavailable",
602+
message: preparationFailure.message,
603+
},
604+
});
605+
}).pipe(Effect.provide(TestClock.layer())),
606+
);
607+
608+
it.effect("times out an SSH route that stops answering after a slow preparation", () =>
609+
Effect.gen(function* () {
610+
const harness = yield* makeHarness({
611+
prepare: () => Effect.sleep("4 minutes").pipe(Effect.as(PREPARED_CONNECTION)),
612+
ready: () => Effect.never,
613+
});
614+
const supervisor = yield* EnvironmentSupervisor.make(SSH_ENTRY, {
615+
initiallyDesired: true,
616+
}).pipe(Effect.provide(harness.dependencies));
617+
618+
yield* awaitState(
619+
supervisor.state,
620+
(state) => state.phase === "connecting" && state.stage === "preparing",
621+
);
622+
yield* TestClock.adjust("4 minutes");
623+
yield* awaitState(
624+
supervisor.state,
625+
(state) => state.phase === "connecting" && state.stage === "synchronizing",
626+
);
627+
const firstFailure = yield* awaitState(
628+
supervisor.state,
629+
(state) => state.phase === "backoff",
630+
).pipe(Effect.forkChild({ startImmediately: true }));
631+
yield* TestClock.adjust("90 seconds");
632+
633+
expect(yield* Fiber.join(firstFailure)).toMatchObject({
634+
attempt: 1,
635+
lastFailure: {
636+
_tag: "ConnectionTransientError",
637+
reason: "timeout",
638+
message: "Test environment did not respond during connection setup.",
639+
},
640+
});
641+
}).pipe(Effect.provide(TestClock.layer())),
642+
);
643+
644+
it.effect("does not time a second SSH route's slow preparation", () =>
645+
Effect.gen(function* () {
646+
const secondSshTarget = new SshConnectionTarget({
647+
environmentId: TARGET.environmentId,
648+
label: TARGET.label,
649+
connectionId: "ssh-connection-2",
650+
});
651+
const firstRouteFailure = transient("The first SSH route dropped while synchronizing.");
652+
const harness = yield* makeHarness({
653+
prepare: (attempt, target) =>
654+
attempt === 1
655+
? Effect.succeed(preparedFor(target))
656+
: Effect.sleep("4 minutes").pipe(
657+
Effect.andThen(
658+
Effect.fail(
659+
new ConnectionTransientError({
660+
reason: "remote-unavailable",
661+
detail: "Could not prepare the SSH environment.",
662+
}),
663+
),
664+
),
665+
),
666+
ready: () => Effect.fail(firstRouteFailure),
667+
});
668+
const supervisor = yield* EnvironmentSupervisor.make(
669+
{ ...SSH_ENTRY, alternateRoutes: [{ target: secondSshTarget, profile: Option.none() }] },
670+
{ initiallyDesired: true },
671+
).pipe(Effect.provide(harness.dependencies));
672+
673+
yield* awaitState(
674+
supervisor.state,
675+
(state) => state.phase === "connecting" && state.stage === "preparing",
676+
);
677+
const firstFailure = yield* awaitState(
678+
supervisor.state,
679+
(state) => state.phase === "backoff",
680+
).pipe(Effect.forkChild({ startImmediately: true }));
681+
yield* TestClock.adjust("4 minutes");
682+
683+
// The driver reports the first transient failure across its routes.
684+
expect(yield* Fiber.join(firstFailure)).toMatchObject({
685+
attempt: 1,
686+
lastFailure: { reason: "transport", message: firstRouteFailure.message },
687+
});
688+
expect(yield* Ref.get(harness.prepareCount)).toBe(2);
689+
}).pipe(Effect.provide(TestClock.layer())),
690+
);
691+
559692
it.effect("treats an involuntary session close as transient and reconnects", () =>
560693
Effect.gen(function* () {
561694
const harness = yield* makeHarness();

‎packages/client-runtime/src/connection/supervisor.ts‎

Lines changed: 18 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -909,18 +909,29 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
909909
// each route that reaches synchronizing adds its own snapshot budget, so
910910
// a route that fails late does not eat the next route's time.
911911
Effect.gen(function* () {
912-
const routeCount = connectionRoutes(yield* Ref.get(currentEntry)).length;
913-
const startedAt = yield* Clock.monotonicTimeNanos;
914-
const setup = Duration.toNanosUnsafe(Duration.times(establishmentTimeout, routeCount));
912+
const routes = connectionRoutes(yield* Ref.get(currentEntry));
913+
const setup = Duration.toNanosUnsafe(Duration.times(establishmentTimeout, routes.length));
914+
// The desktop bounds SSH preparation, which may download a server
915+
// release, so for SSH-only entries the clock stops while preparing.
916+
const pausesWhilePreparing = routes.every((route) => connectionRouteKind(route) === "ssh");
917+
const preparing = (expected: boolean) =>
918+
SubscriptionRef.changes(state).pipe(
919+
Stream.filter((current) => (current.stage === "preparing") === expected),
920+
Stream.runHead,
921+
);
922+
let spent = 0n;
915923
while (true) {
924+
if (pausesWhilePreparing) yield* preparing(false);
916925
const synchronized = BigInt(yield* Ref.get(synchronizedRoutes));
917-
const deadline =
918-
startedAt + setup + synchronized * Duration.toNanosUnsafe(synchronizationTimeout);
919-
const remaining = deadline - (yield* Clock.monotonicTimeNanos);
926+
const remaining =
927+
setup + synchronized * Duration.toNanosUnsafe(synchronizationTimeout) - spent;
920928
if (remaining <= 0n) {
921929
return { _tag: "TimedOut" } as const satisfies EstablishmentEvent;
922930
}
923-
yield* Effect.sleep(Duration.nanos(remaining));
931+
const resumedAt = yield* Clock.monotonicTimeNanos;
932+
const sleep = Effect.sleep(Duration.nanos(remaining));
933+
yield* pausesWhilePreparing ? Effect.raceFirst(sleep, preparing(true)) : sleep;
934+
spent += (yield* Clock.monotonicTimeNanos) - resumedAt;
924935
}
925936
}),
926937
]);

0 commit comments

Comments
 (0)