Skip to content

Commit 990bb0b

Browse files
t3dotggclaude
andauthored
fix: reconnect faster after remote server updates (pingdotgg#5404)
Co-authored-by: Claude Fable 5 <[email protected]>
1 parent 7251f1a commit 990bb0b

3 files changed

Lines changed: 98 additions & 14 deletions

File tree

‎apps/server/src/server.ts‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -583,8 +583,13 @@ export const makeServerLayer = Layer.unwrap(
583583
const server = yield* HttpServer.HttpServer;
584584
const address = server.address;
585585
if (typeof address === "string" || !("port" in address)) return;
586-
yield* Effect.sleep("250 millis").pipe(
587-
Effect.andThen(reconcileDesiredCloudLink(`http://127.0.0.1:${address.port}`)),
586+
// No settling delay before the first attempt: routes are already
587+
// serving by the time activation opens this gate (the startup
588+
// sequence awaits routesReady), and the retry schedule below
589+
// covers anything this sleep used to hedge against. Every
590+
// millisecond here is dead time on the path to remote
591+
// reachability after a restart.
592+
yield* reconcileDesiredCloudLink(`http://127.0.0.1:${address.port}`).pipe(
588593
Effect.retry({
589594
while: (error) =>
590595
error._tag !== "EnvironmentHttpBadRequestError" &&

‎packages/client-runtime/src/state/server.test.ts‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,16 @@ import {
77
} from "@t3tools/contracts";
88
import { describe, expect, it } from "@effect/vitest";
99
import * as Cause from "effect/Cause";
10+
import * as Duration from "effect/Duration";
1011
import * as Effect from "effect/Effect";
1112
import * as Exit from "effect/Exit";
13+
import * as Fiber from "effect/Fiber";
1214
import * as Option from "effect/Option";
1315
import * as Queue from "effect/Queue";
16+
import * as Ref from "effect/Ref";
1417
import * as Stream from "effect/Stream";
1518
import * as SubscriptionRef from "effect/SubscriptionRef";
19+
import * as TestClock from "effect/testing/TestClock";
1620
import { RpcClientError } from "effect/unstable/rpc";
1721
import * as Socket from "effect/unstable/socket/Socket";
1822

@@ -30,6 +34,7 @@ import {
3034
makeEnvironmentServerConfigState,
3135
isLegacyUpdateHandoffLoss,
3236
matchesServerUpdateReadyEvent,
37+
nudgeReconnectDuringUpdateRestart,
3338
projectServerWelcome,
3439
resolveServerConfigValue,
3540
resolveServerUpdateProgressResult,
@@ -71,6 +76,56 @@ function session(client: WsRpcProtocolClient): RpcSession {
7176
};
7277
}
7378

79+
describe("update restart reconnect nudges", () => {
80+
it.effect("retries once per backoff entry instead of only the first", () =>
81+
Effect.gen(function* () {
82+
const retries = yield* Ref.make(0);
83+
const states = [
84+
{ phase: "backoff" },
85+
{ phase: "connecting" },
86+
{ phase: "backoff" },
87+
{ phase: "backoff" },
88+
];
89+
90+
yield* nudgeReconnectDuringUpdateRestart({
91+
stateChanges: Stream.fromIterable(states),
92+
retryNow: Ref.update(retries, (count) => count + 1),
93+
interval: Duration.zero,
94+
});
95+
96+
// Three backoff entries, three nudges. The old one-shot behavior fired
97+
// once and then let the supervisor's ladder stretch to 16-second gaps.
98+
expect(yield* Ref.get(retries)).toBe(3);
99+
}),
100+
);
101+
102+
it.effect("paces nudges so a fast-failing connection cannot spin", () =>
103+
Effect.gen(function* () {
104+
const retries = yield* Ref.make(0);
105+
106+
const fiber = yield* Effect.forkChild(
107+
nudgeReconnectDuringUpdateRestart({
108+
stateChanges: Stream.fromIterable([{ phase: "backoff" }, { phase: "backoff" }]),
109+
retryNow: Ref.update(retries, (count) => count + 1),
110+
}),
111+
{ startImmediately: true },
112+
);
113+
114+
// Each nudge waits out the interval first, so nothing fires immediately.
115+
yield* TestClock.adjust(Duration.millis(999));
116+
expect(yield* Ref.get(retries)).toBe(0);
117+
118+
yield* TestClock.adjust(Duration.millis(1));
119+
expect(yield* Ref.get(retries)).toBe(1);
120+
121+
yield* TestClock.adjust(Duration.seconds(1));
122+
expect(yield* Ref.get(retries)).toBe(2);
123+
124+
yield* Fiber.join(fiber);
125+
}).pipe(Effect.provide(TestClock.layer())),
126+
);
127+
});
128+
74129
describe("server state projection", () => {
75130
it("only treats a legacy transport interruption as an unacknowledged handoff", () => {
76131
expect(isLegacyUpdateHandoffLoss(Cause.interrupt(1))).toBe(true);

‎packages/client-runtime/src/state/server.ts‎

Lines changed: 36 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -149,6 +149,35 @@ export function validateServerUpdateReadyEvent(
149149
);
150150
}
151151

152+
/**
153+
* Keeps reconnect attempts ~1s apart for the whole update restart.
154+
*
155+
* A restart takes the server down for ~15 seconds, but the supervisor's normal
156+
* backoff ladder (1/2/4/8/16s) assumes an unexpected failure and lands attempts
157+
* at ~3, 5, 9, 17 and 33 seconds — so a 15-second restart is observed as a
158+
* 33-second "Resuming". Nudging on every backoff entry (not just the first)
159+
* holds the retry cadence flat until the server answers again. The sleep before
160+
* each nudge is the pacer: a connection that fails instantly re-enters backoff
161+
* immediately and would otherwise spin a tight retry loop.
162+
*
163+
* Callers fork this as a child of the update command so it is interrupted as
164+
* soon as the update settles, whether it succeeds, fails, or times out.
165+
*/
166+
export function nudgeReconnectDuringUpdateRestart(input: {
167+
readonly stateChanges: Stream.Stream<{ readonly phase: string }, unknown>;
168+
readonly retryNow: Effect.Effect<void>;
169+
readonly interval?: Duration.Duration;
170+
}): Effect.Effect<void> {
171+
return input.stateChanges.pipe(
172+
Stream.filter((state) => state.phase === "backoff"),
173+
Stream.runForEach(() =>
174+
Effect.sleep(input.interval ?? Duration.seconds(1)).pipe(Effect.andThen(input.retryNow)),
175+
),
176+
Effect.timeoutOption(SERVER_UPDATE_RESUME_TIMEOUT),
177+
Effect.ignore,
178+
);
179+
}
180+
152181
export function serverUpdateStateForProgressEvent(
153182
fromVersion: string,
154183
targetVersion: string,
@@ -589,18 +618,13 @@ export function createServerEnvironmentAtoms<R, E>(
589618
}),
590619
);
591620

592-
// The update restart is intentional. As soon as the supervisor sees
593-
// that first failed connection, discard any prior backoff debt and
594-
// retry immediately instead of carrying an old 16-second delay.
595-
yield* environmentRegistry.stateChanges(target.environmentId).pipe(
596-
Stream.filter((state) => state.phase === "backoff"),
597-
Stream.take(1),
598-
Stream.runDrain,
599-
Effect.andThen(environmentRegistry.retryNow(target.environmentId)),
600-
Effect.timeoutOption(Duration.seconds(30)),
601-
Effect.ignore,
602-
Effect.forkChild,
603-
);
621+
// The update restart is intentional and the server stays unreachable
622+
// for the whole restart, so hold the retry cadence flat instead of
623+
// letting the supervisor climb its backoff ladder.
624+
yield* nudgeReconnectDuringUpdateRestart({
625+
stateChanges: environmentRegistry.stateChanges(target.environmentId),
626+
retryNow: environmentRegistry.retryNow(target.environmentId),
627+
}).pipe(Effect.forkChild);
604628

605629
const resumed = yield* environmentRegistry
606630
.followStream(target.environmentId, subscribe(WS_METHODS.subscribeServerLifecycle, {}))

0 commit comments

Comments
 (0)