Skip to content

Commit d062787

Browse files
fix(client): back off better-route checks of silent routes
A connection on a fallback route preflighted every better route each minute. A saved LAN route that never answers waited out the 2.5s probe timeout every time, ~1k probes a day. Failed checks now back off from 2 up to 30 minutes per route, and any network change or long resume clears the backoff so a route that comes back is still found at once. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 08a463e commit d062787

2 files changed

Lines changed: 90 additions & 5 deletions

File tree

‎packages/client-runtime/src/connection/supervisor.test.ts‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1614,6 +1614,48 @@ describe("EnvironmentSupervisor routes", () => {
16141614
}),
16151615
);
16161616

1617+
it.effect("checks a dead LAN route less often until the network changes", () =>
1618+
Effect.gen(function* () {
1619+
const lanReachable = yield* Ref.make(false);
1620+
const lanChecks = yield* Ref.make(0);
1621+
const harness = yield* makeHarness({
1622+
checkRoute: (route) =>
1623+
route.target._tag === "BearerConnectionTarget"
1624+
? Ref.update(lanChecks, (count) => count + 1).pipe(
1625+
Effect.andThen(Ref.get(lanReachable)),
1626+
Effect.map((reachable) => (reachable ? "answered" : "silent")),
1627+
)
1628+
: Effect.succeed("unchecked"),
1629+
prepare: (_attempt, target) => Effect.succeed(preparedFor(target)),
1630+
});
1631+
const supervisor = yield* EnvironmentSupervisor.make(LAN_THEN_RELAY_ENTRY, {
1632+
initiallyDesired: true,
1633+
}).pipe(Effect.provide(harness.dependencies));
1634+
1635+
yield* awaitState(
1636+
supervisor.state,
1637+
(state) => state.phase === "connected" && state.generation === 1,
1638+
);
1639+
yield* Ref.set(lanChecks, 0);
1640+
1641+
// Checked at 1, 3 and 7 minutes instead of every minute.
1642+
for (let minute = 0; minute < 10; minute += 1) {
1643+
yield* TestClock.adjust("60 seconds");
1644+
}
1645+
expect(yield* Ref.get(lanChecks)).toBe(3);
1646+
1647+
yield* Ref.set(lanReachable, true);
1648+
yield* harness.wake("network-changed");
1649+
yield* awaitState(
1650+
supervisor.state,
1651+
(state) => state.phase === "connected" && state.generation === 2,
1652+
);
1653+
expect(Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared)).target._tag).toBe(
1654+
"BearerConnectionTarget",
1655+
);
1656+
}),
1657+
);
1658+
16171659
it.effect("moves off a dead LAN socket when the phone switches to cellular", () =>
16181660
Effect.gen(function* () {
16191661
const lanReachable = yield* Ref.make(true);

‎packages/client-runtime/src/connection/supervisor.ts‎

Lines changed: 48 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,11 @@ const SILENT_SOCKET_RECHECK_EVERY = "1 second";
5858
// instead of timing out.
5959
const REMOTE_SILENT_SOCKET_AFTER = "2500 millis";
6060
const BACKOFF_RESET_AFTER_MS = 30_000;
61-
const BETTER_ROUTE_CHECK_INTERVAL = "60 seconds";
61+
const BETTER_ROUTE_CHECK_INTERVAL_MS = 60_000;
62+
// Each check of a dead route can wait out the full probe timeout, so a route
63+
// that keeps failing is checked half as often each time, until the network
64+
// changes and it may answer again.
65+
const SILENT_ROUTE_BACKOFF_MAX_MS = 1_800_000;
6266
// A better route that answered the check but then failed to connect is not
6367
// tried again for this long, so a flaky LAN cannot bounce the connection.
6468
const BETTER_ROUTE_COOLDOWN_MS = 300_000;
@@ -139,6 +143,20 @@ function retryDelayMs(failureCount: number, random: number): number {
139143
return Math.round(ceiling / 2 + (ceiling / 2) * random);
140144
}
141145

146+
interface SilentRoute {
147+
readonly failures: number;
148+
readonly until: bigint;
149+
}
150+
151+
function backOffSilentRoute(previous: SilentRoute | undefined, now: bigint): SilentRoute {
152+
const failures = (previous?.failures ?? 0) + 1;
153+
const delayMs = Math.min(
154+
SILENT_ROUTE_BACKOFF_MAX_MS,
155+
BETTER_ROUTE_CHECK_INTERVAL_MS * 2 ** failures,
156+
);
157+
return { failures, until: now + BigInt(delayMs) * 1_000_000n };
158+
}
159+
142160
function annotateTarget(target: ConnectionTarget) {
143161
return Effect.annotateCurrentSpan({
144162
"environment.id": target.environmentId,
@@ -288,6 +306,8 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
288306
const currentRoutes = Ref.get(currentEntry).pipe(Effect.map(connectionRoutes));
289307
const preferredRouteId = yield* Ref.make(Option.none<string>());
290308
const routeCooldowns = yield* Ref.make<ReadonlyMap<string, bigint>>(new Map());
309+
const silentRoutes = yield* Ref.make<ReadonlyMap<string, SilentRoute>>(new Map());
310+
const forgetSilentRoutes = Ref.set(silentRoutes, new Map());
291311

292312
const attemptEntry = Effect.gen(function* () {
293313
const latest = yield* Ref.get(currentEntry);
@@ -338,14 +358,27 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
338358
if (current <= 0) return;
339359
const now = yield* Clock.monotonicTimeNanos;
340360
const cooldowns = yield* Ref.get(routeCooldowns);
361+
const silent = yield* Ref.get(silentRoutes);
341362
const better = connectionRoutes(latest)
342363
.slice(0, current)
343-
.filter((route) => (cooldowns.get(connectionRouteId(route.target)) ?? 0n) <= now);
364+
.filter((route) => {
365+
const id = connectionRouteId(route.target);
366+
return (cooldowns.get(id) ?? 0n) <= now && (silent.get(id)?.until ?? 0n) <= now;
367+
});
344368
const passed = yield* Effect.forEach(
345369
better,
346370
(route: ConnectionRoute) => driver.preflight(latest, route),
347371
{ concurrency: "unbounded" },
348372
);
373+
yield* Ref.update(silentRoutes, (previous) => {
374+
const next = new Map(previous);
375+
for (const [position, route] of better.entries()) {
376+
const id = connectionRouteId(route.target);
377+
if (passed[position]) next.delete(id);
378+
else next.set(id, backOffSilentRoute(previous.get(id), now));
379+
}
380+
return next;
381+
});
349382
const index = passed.indexOf(true);
350383
if (index === -1) return;
351384
const live = yield* SubscriptionRef.get(session);
@@ -668,7 +701,7 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
668701
);
669702
// Ticks always run: the check skips the preferred route itself, and the
670703
// route list can grow while connected (learned routes).
671-
yield* Stream.tick(BETTER_ROUTE_CHECK_INTERVAL).pipe(
704+
yield* Stream.tick(Duration.millis(BETTER_ROUTE_CHECK_INTERVAL_MS)).pipe(
672705
Stream.drop(1),
673706
Stream.runForEach(() => requestBetterRouteCheck(lease)),
674707
Effect.forkScoped,
@@ -1114,14 +1147,24 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
11141147
current.network === network ? [false, current] : ([true, { ...current, network }] as const),
11151148
).pipe(
11161149
Effect.flatMap((changed) =>
1117-
changed ? signal({ _tag: "NetworkChanged", network }) : Effect.void,
1150+
changed
1151+
? (network === "offline" ? Effect.void : forgetSilentRoutes).pipe(
1152+
Effect.andThen(signal({ _tag: "NetworkChanged", network })),
1153+
)
1154+
: Effect.void,
11181155
),
11191156
),
11201157
),
11211158
Effect.forkScoped,
11221159
);
11231160
yield* wakeups.changes.pipe(
1124-
Stream.runForEach((reason) => signal({ _tag: "Wakeup", reason })),
1161+
Stream.runForEach((reason) =>
1162+
// A long suspension can hide a network change the platform never reported.
1163+
(reason === "network-changed" || reason === "application-active-long"
1164+
? forgetSilentRoutes
1165+
: Effect.void
1166+
).pipe(Effect.andThen(signal({ _tag: "Wakeup", reason }))),
1167+
),
11251168
Effect.forkScoped,
11261169
);
11271170
yield* Queue.take(betterRouteChecks).pipe(

0 commit comments

Comments
 (0)