Skip to content

Commit 5bba764

Browse files
saphidgithub-actions[bot]claude
authored
fix(client-runtime): slow config snapshots no longer drop working connections (pingdotgg#14518)
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Claude Sonnet 5.5 <[email protected]> Co-authored-by: SAPHID <[email protected]>
1 parent 94ca023 commit 5bba764

2 files changed

Lines changed: 98 additions & 9 deletions

File tree

‎packages/client-runtime/src/connection/supervisor.test.ts‎

Lines changed: 57 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -488,6 +488,29 @@ describe("EnvironmentSupervisor", () => {
488488
}).pipe(Effect.provide(TestClock.layer())),
489489
);
490490

491+
it.effect("keeps a working socket open while a slow server config snapshot arrives", () =>
492+
Effect.gen(function* () {
493+
const harness = yield* makeHarness({
494+
ready: () => Effect.sleep("40 seconds"),
495+
});
496+
const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
497+
initiallyDesired: true,
498+
}).pipe(Effect.provide(harness.dependencies));
499+
500+
yield* awaitState(
501+
supervisor.state,
502+
(state) => state.phase === "connecting" && state.stage === "synchronizing",
503+
);
504+
yield* TestClock.adjust("39 seconds");
505+
expect((yield* SubscriptionRef.get(supervisor.state)).phase).toBe("connecting");
506+
507+
yield* TestClock.adjust("1 second");
508+
yield* awaitState(supervisor.state, (state) => state.phase === "connected");
509+
expect(yield* Ref.get(harness.prepareCount)).toBe(1);
510+
expect(yield* Ref.get(harness.releaseCount)).toBe(0);
511+
}).pipe(Effect.provide(TestClock.layer())),
512+
);
513+
491514
it.effect("retries when a session never becomes ready", () =>
492515
Effect.gen(function* () {
493516
const harness = yield* makeHarness({
@@ -501,7 +524,8 @@ describe("EnvironmentSupervisor", () => {
501524
supervisor.state,
502525
(state) => state.phase === "connecting" && state.stage === "synchronizing",
503526
);
504-
yield* TestClock.adjust("14 seconds");
527+
// 15 seconds of setup plus 75 for the snapshot; setup finished instantly.
528+
yield* TestClock.adjust("89 seconds");
505529
expect((yield* SubscriptionRef.get(supervisor.state)).stage).toBe("synchronizing");
506530

507531
yield* TestClock.adjust("1 second");
@@ -1742,6 +1766,38 @@ describe("EnvironmentSupervisor routes", () => {
17421766
}),
17431767
);
17441768

1769+
it.effect(
1770+
"gives the next route its own snapshot budget after a route fails while synchronizing",
1771+
() =>
1772+
Effect.gen(function* () {
1773+
const harness = yield* makeHarness({
1774+
checkRoute: () => Effect.succeed("answered"),
1775+
prepare: (_attempt, target) => Effect.succeed(preparedFor(target)),
1776+
// The LAN session fails 50 seconds into its snapshot; T3 Connect needs 40.
1777+
ready: (attempt) =>
1778+
attempt === 1
1779+
? Effect.sleep("50 seconds").pipe(Effect.andThen(Effect.fail(transient())))
1780+
: Effect.sleep("40 seconds"),
1781+
});
1782+
const supervisor = yield* EnvironmentSupervisor.make(LAN_THEN_RELAY_ENTRY, {
1783+
initiallyDesired: true,
1784+
}).pipe(Effect.provide(harness.dependencies));
1785+
1786+
yield* awaitState(
1787+
supervisor.state,
1788+
(state) => state.phase === "connecting" && state.stage === "synchronizing",
1789+
);
1790+
yield* TestClock.adjust("50 seconds");
1791+
yield* TestClock.adjust("40 seconds");
1792+
yield* awaitState(supervisor.state, (state) => state.phase === "connected");
1793+
expect(Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared)).target._tag).toBe(
1794+
"RelayConnectionTarget",
1795+
);
1796+
expect(yield* Ref.get(harness.prepareCount)).toBe(2);
1797+
expect(yield* Ref.get(harness.releaseCount)).toBe(1);
1798+
}).pipe(Effect.provide(TestClock.layer())),
1799+
);
1800+
17451801
it.effect("still tries a silent LAN route after the T3 Connect route fails", () =>
17461802
Effect.gen(function* () {
17471803
const harness = yield* makeHarness({

‎packages/client-runtime/src/connection/supervisor.ts‎

Lines changed: 41 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -39,8 +39,15 @@ import {
3939

4040
const RETRY_BASE_DELAY_MS = 1_000;
4141
const RETRY_MAX_DELAY_MS = 300_000;
42+
// Per-route budget to resolve credentials and create the session, which ends
43+
// when the driver reports the synchronizing stage. Each route that gets that
44+
// far adds its own budget for the socket to open (bounded at 15 seconds by the
45+
// RPC session) and the server to deliver the first config snapshot. A cold
46+
// server with a large database can take longer than 15 seconds to answer, and
47+
// that is not a reason to drop a working connection.
4248
const CONNECTION_ESTABLISHMENT_TIMEOUT = "15 seconds";
4349
const establishmentTimeout = Duration.fromInputUnsafe(CONNECTION_ESTABLISHMENT_TIMEOUT);
50+
const synchronizationTimeout = Duration.fromInputUnsafe("75 seconds");
4451
const CONNECTION_PROBE_TIMEOUT = "15 seconds";
4552
// Mobile resumes, explicit retries, and offline events want a fast answer:
4653
// the user is waiting, or the network may be gone.
@@ -428,9 +435,13 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
428435
attempt: number,
429436
generation: number,
430437
lastFailure: ConnectionAttemptError | null,
438+
synchronizedRoutes: Ref.Ref<number>,
431439
) {
432440
return yield* driver.connect(yield* attemptEntry, (progress) =>
433-
reportProgress(attempt, generation, lastFailure, progress),
441+
(progress.stage === "synchronizing"
442+
? Ref.update(synchronizedRoutes, (count) => count + 1)
443+
: Effect.void
444+
).pipe(Effect.andThen(reportProgress(attempt, generation, lastFailure, progress))),
434445
);
435446
});
436447

@@ -481,16 +492,17 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
481492
generation: number,
482493
lastFailure: ConnectionAttemptError | null,
483494
pendingRetry: Option.Option<PendingRetryTrace>,
495+
synchronizedRoutes: Ref.Ref<number>,
484496
) {
485497
if (usesRelay) {
486498
return yield* traceRelayEstablishment(
487-
establishConnection(attempt, generation, lastFailure),
499+
establishConnection(attempt, generation, lastFailure, synchronizedRoutes),
488500
attempt,
489501
generation,
490502
pendingRetry,
491503
);
492504
}
493-
return yield* establishConnection(attempt, generation, lastFailure).pipe(
505+
return yield* establishConnection(attempt, generation, lastFailure, synchronizedRoutes).pipe(
494506
Effect.map((lease) => ({
495507
attemptSpan: Option.none<Tracer.Span>(),
496508
lease,
@@ -684,9 +696,16 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
684696
) {
685697
const switchingTo = yield* Ref.get(preferredRouteId);
686698
yield* SubscriptionRef.set(prepared, Option.none());
699+
const synchronizedRoutes = yield* Ref.make(0);
687700
const establishment = yield* Effect.raceAllFirst([
688701
exitUnlessInterrupted(
689-
establishTracedConnection(attempt, generation, lastFailure, pendingRetry),
702+
establishTracedConnection(
703+
attempt,
704+
generation,
705+
lastFailure,
706+
pendingRetry,
707+
synchronizedRoutes,
708+
),
690709
).pipe(
691710
Effect.map((exit): EstablishmentEvent => ({
692711
_tag: "Completed",
@@ -699,10 +718,24 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
699718
resetRetry,
700719
})),
701720
),
702-
// Each route may use the full setup time before the next is tried.
703-
Effect.sleep(
704-
Duration.times(establishmentTimeout, connectionRoutes(yield* Ref.get(currentEntry)).length),
705-
).pipe(Effect.as<EstablishmentEvent>({ _tag: "TimedOut" })),
721+
// Each route may use the full setup time before the next is tried, and
722+
// each route that reaches synchronizing adds its own snapshot budget, so
723+
// a route that fails late does not eat the next route's time.
724+
Effect.gen(function* () {
725+
const routeCount = connectionRoutes(yield* Ref.get(currentEntry)).length;
726+
const startedAt = yield* Clock.monotonicTimeNanos;
727+
const setup = Duration.toNanosUnsafe(Duration.times(establishmentTimeout, routeCount));
728+
while (true) {
729+
const synchronized = BigInt(yield* Ref.get(synchronizedRoutes));
730+
const deadline =
731+
startedAt + setup + synchronized * Duration.toNanosUnsafe(synchronizationTimeout);
732+
const remaining = deadline - (yield* Clock.monotonicTimeNanos);
733+
if (remaining <= 0n) {
734+
return { _tag: "TimedOut" } as const satisfies EstablishmentEvent;
735+
}
736+
yield* Effect.sleep(Duration.nanos(remaining));
737+
}
738+
}),
706739
]);
707740

708741
if (establishment._tag === "Interrupted") {

0 commit comments

Comments
 (0)