Skip to content

Commit 2db8ba7

Browse files
t3dotggclaude
andcommitted
fix: drop a promoted route that fails after preparation
The relay broker only reports override failures inside prepare. A direct route that opened but died during socket setup, before readiness, or later in the session stayed installed, so every retry re-selected the same broken route instead of falling back to the relay. The supervisor now reports those failures too, which clears the override and starts its cooldown. Co-Authored-By: Claude Fable 5.1 <[email protected]>
1 parent 6ee5396 commit 2db8ba7

2 files changed

Lines changed: 80 additions & 2 deletions

File tree

‎packages/client-runtime/src/connection/supervisor.test.ts‎

Lines changed: 58 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,8 @@ const makeHarness = Effect.fn("TestConnectionHarness.make")(function* (options?:
119119
readonly discover?: (
120120
prepared: PreparedConnection,
121121
) => Effect.Effect<Option.Option<ConnectionPromotion.PromotedRoute>>;
122+
readonly overrideFor?: () => Effect.Effect<Option.Option<ConnectionPromotion.PromotedRoute>>;
123+
readonly reportOverrideFailed?: () => Effect.Effect<void>;
122124
}) {
123125
const networkStatus = yield* SubscriptionRef.make<NetworkStatus>(
124126
options?.networkStatus ?? "online",
@@ -202,8 +204,8 @@ const makeHarness = Effect.fn("TestConnectionHarness.make")(function* (options?:
202204
Layer.succeed(
203205
ConnectionPromotion.ConnectionPromotion,
204206
ConnectionPromotion.ConnectionPromotion.of({
205-
overrideFor: () => Effect.succeed(Option.none()),
206-
reportOverrideFailed: () => Effect.void,
207+
overrideFor: options?.overrideFor ?? (() => Effect.succeed(Option.none())),
208+
reportOverrideFailed: options?.reportOverrideFailed ?? (() => Effect.void),
207209
discover,
208210
}),
209211
),
@@ -916,6 +918,60 @@ describe("EnvironmentSupervisor", () => {
916918
}),
917919
);
918920

921+
it.effect("reports a promoted route that fails after preparation so retries fall back", () =>
922+
Effect.gen(function* () {
923+
const directRoute: ConnectionPromotion.PromotedRoute = {
924+
endpointId: "server-lan:http://192.168.1.20:3773",
925+
httpBaseUrl: "http://192.168.1.20:3773/",
926+
wsBaseUrl: "ws://192.168.1.20:3773/",
927+
};
928+
const relayPrepared: PreparedConnection = {
929+
environmentId: RELAY_TARGET.environmentId,
930+
label: RELAY_TARGET.label,
931+
httpBaseUrl: "https://tunnel.example.test",
932+
socketUrl: "wss://tunnel.example.test/ws?wsTicket=dpop",
933+
httpAuthorization: { _tag: "Dpop", accessToken: "dpop-access-token" },
934+
target: RELAY_TARGET,
935+
};
936+
const directPrepared: PreparedConnection = {
937+
...relayPrepared,
938+
httpBaseUrl: directRoute.httpBaseUrl,
939+
socketUrl: "ws://192.168.1.20:3773/ws?wsTicket=direct",
940+
};
941+
const override = yield* Ref.make<Option.Option<ConnectionPromotion.PromotedRoute>>(
942+
Option.some(directRoute),
943+
);
944+
const failureReports = yield* Ref.make(0);
945+
const harness = yield* makeHarness({
946+
// The override is installed up front, so the first attempt prepares
947+
// the direct route; once the override is cleared the retry relays.
948+
prepare: () =>
949+
Ref.get(override).pipe(
950+
Effect.map((current) => (Option.isSome(current) ? directPrepared : relayPrepared)),
951+
),
952+
// The direct socket opens but never becomes ready.
953+
ready: (attempt) =>
954+
attempt === 1 ? Effect.fail(transient("direct socket closed before ready")) : Effect.void,
955+
discover: () => Effect.succeed(Option.none()),
956+
overrideFor: () => Ref.get(override),
957+
reportOverrideFailed: () =>
958+
Ref.set(override, Option.none()).pipe(
959+
Effect.andThen(Ref.update(failureReports, (count) => count + 1)),
960+
),
961+
});
962+
const supervisor = yield* EnvironmentSupervisor.make(RELAY_ENTRY, {
963+
initiallyDesired: true,
964+
}).pipe(Effect.provide(harness.dependencies));
965+
966+
yield* awaitState(supervisor.state, (state) => state.phase === "backoff");
967+
expect(yield* Ref.get(failureReports)).toBe(1);
968+
yield* TestClock.adjust("3 seconds");
969+
yield* awaitState(supervisor.state, (state) => state.phase === "connected");
970+
const prepared = yield* SubscriptionRef.get(supervisor.prepared);
971+
expect(Option.getOrThrow(prepared).httpBaseUrl).toBe("https://tunnel.example.test");
972+
}).pipe(Effect.provide(TestClock.layer())),
973+
);
974+
919975
it.effect("does not run direct-route discovery for non-relay targets", () =>
920976
Effect.gen(function* () {
921977
const discoveries = yield* Ref.make(0);

‎packages/client-runtime/src/connection/supervisor.ts‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -412,6 +412,25 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
412412
}).pipe(Effect.forkScoped);
413413
});
414414

415+
// A promoted direct route can pass preparation and still die while the
416+
// socket opens, before readiness, or later in the session. The broker only
417+
// learns about failures inside prepare, so the supervisor reports the rest:
418+
// clear the override and start its cooldown so the next attempt falls back
419+
// to the relay instead of re-selecting the same broken route.
420+
const reportFailedPromotedRoute = Effect.fnUntraced(function* () {
421+
if (Option.isNone(promotion) || target._tag !== "RelayConnectionTarget") {
422+
return;
423+
}
424+
const attempted = yield* SubscriptionRef.get(prepared);
425+
if (Option.isNone(attempted)) {
426+
return;
427+
}
428+
const override = yield* promotion.value.overrideFor(target.environmentId);
429+
if (Option.isSome(override) && override.value.httpBaseUrl === attempted.value.httpBaseUrl) {
430+
yield* promotion.value.reportOverrideFailed(target.environmentId);
431+
}
432+
});
433+
415434
const waitForEstablishmentInterrupt = Effect.fnUntraced(function* () {
416435
for (;;) {
417436
const next = yield* Queue.take(signals);
@@ -581,6 +600,7 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
581600
} satisfies AttemptOutcome;
582601
}
583602
if (establishment._tag === "TimedOut") {
603+
yield* reportFailedPromotedRoute();
584604
return {
585605
_tag: "Failure",
586606
established: false,
@@ -595,6 +615,7 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
595615
} satisfies AttemptOutcome;
596616
}
597617
if (Exit.isFailure(establishment.exit)) {
618+
yield* reportFailedPromotedRoute();
598619
const isUnexpectedDefect =
599620
!Cause.hasInterruptsOnly(establishment.exit.cause) &&
600621
!establishment.exit.cause.reasons.some(Cause.isFailReason);
@@ -666,6 +687,7 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
666687
resetRetry: connectedExit.value,
667688
} satisfies AttemptOutcome;
668689
}
690+
yield* reportFailedPromotedRoute();
669691
return failureFromExit(target, connectedExit, true, connectedForMs >= BACKOFF_RESET_AFTER_MS);
670692
}, Effect.ensuring(clearLease));
671693

0 commit comments

Comments
 (0)