Skip to content

Commit 02343e8

Browse files
Jgratton24claude
andcommitted
fix(desktop): cap consecutive never-ready backend exits and surface instead of restarting forever
Ports the still-relevant piece of #3623: a backend that spawns fine but keeps exiting before it ever answers a readiness probe previously restarted silently forever (500ms-10s backoff, no cap, no surfacing). After 5 consecutive never-ready exits the failure now surfaces through onPreflightFailed — dialog + Windows fallback on the primary, inline Connections error on the WSL secondary — exactly like exhausted preflight and readiness failures. The counter is scoped strictly to post-spawn exits: pre-spawn preflight retries have their own counter, a crash after the backend was ready resets the streak (an established backend crashing is a different failure than one that never comes up), and the budget refreshes on ready, on external stop, and on a fresh manual start so one cap firing does not permanently remove the retry budget. Co-Authored-By: Claude Fable 5 <[email protected]>
1 parent d3c3ab1 commit 02343e8

2 files changed

Lines changed: 178 additions & 45 deletions

File tree

‎apps/desktop/src/backend/DesktopBackendManager.test.ts‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1590,6 +1590,67 @@ describe("DesktopBackendManager", () => {
15901590
),
15911591
);
15921592

1593+
it.effect("caps runs that keep exiting before ready and surfaces via onPreflightFailed", () =>
1594+
Effect.scoped(
1595+
Effect.gen(function* () {
1596+
const starts = yield* Queue.unbounded<number>();
1597+
const failures = yield* Queue.unbounded<string>();
1598+
const preflightFailures: Array<DesktopBackendManager.PreflightFailure> = [];
1599+
let startCount = 0;
1600+
1601+
const spawnerLayer = Layer.succeed(
1602+
ChildProcessSpawner.ChildProcessSpawner,
1603+
ChildProcessSpawner.make(() =>
1604+
Effect.sync(() => {
1605+
startCount += 1;
1606+
return makeProcess({
1607+
exitCode: Queue.offer(starts, startCount).pipe(
1608+
Effect.as(ChildProcessSpawner.ExitCode(1)),
1609+
),
1610+
});
1611+
}),
1612+
),
1613+
);
1614+
1615+
const instance = yield* makeTestInstance({
1616+
spawnerLayer,
1617+
httpClientLayer: httpClientLayer(() => Effect.never),
1618+
onPreflightFailed: (failure) =>
1619+
Effect.sync(() => {
1620+
preflightFailures.push(failure);
1621+
return false;
1622+
}),
1623+
backendOutputLog: {
1624+
persistFailure: ({ details }) => Queue.offer(failures, details).pipe(Effect.asVoid),
1625+
},
1626+
});
1627+
1628+
yield* instance.start;
1629+
assert.equal(yield* Queue.take(starts), 1);
1630+
yield* Queue.take(failures);
1631+
1632+
// Four more spawn-then-exit-before-ready cycles exhaust the cap.
1633+
for (let attempt = 2; attempt <= 5; attempt += 1) {
1634+
yield* TestClock.adjust(Duration.seconds(15));
1635+
assert.equal(yield* Queue.take(starts), attempt);
1636+
yield* Queue.take(failures);
1637+
}
1638+
1639+
// The fifth never-ready exit surfaces instead of scheduling another
1640+
// silent restart; onPreflightFailed returned false, so the instance
1641+
// stops.
1642+
yield* TestClock.adjust(Duration.seconds(30));
1643+
assert.lengthOf(preflightFailures, 1);
1644+
assert.include(preflightFailures[0]!.reason, "5 times");
1645+
assert.isFalse(preflightFailures[0]!.fatal);
1646+
assert.equal(yield* Queue.size(starts), 0);
1647+
const snapshot = yield* instance.snapshot;
1648+
assert.isFalse(snapshot.desiredRunning);
1649+
assert.isFalse(snapshot.restartScheduled);
1650+
}).pipe(Effect.provide(TestClock.layer())),
1651+
),
1652+
);
1653+
15931654
it.effect("stop reports whether the child fully finalized within the timeout", () =>
15941655
Effect.scoped(
15951656
Effect.gen(function* () {

‎apps/desktop/src/backend/DesktopBackendManager.ts‎

Lines changed: 117 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,12 @@ const MAX_PREFLIGHT_FAILURE_ATTEMPTS = 5;
6565
// onPreflightFailed — a live-but-unreachable backend must not present as an
6666
// indefinite "connecting" state.
6767
const MAX_READINESS_FAILURE_ATTEMPTS = 3;
68+
// After this many consecutive runs that spawned but exited without ever
69+
// becoming ready, surface via onPreflightFailed instead of restarting
70+
// forever. Scoped strictly to post-spawn exits: pre-spawn preflight retries
71+
// (WSL cold-start) have their own counter, and a crash AFTER the backend was
72+
// ready resets the streak — that's a different failure than "never comes up".
73+
const MAX_EXIT_FAILURE_ATTEMPTS = 5;
6874
const DEFAULT_BACKEND_READINESS_TIMEOUT = Duration.minutes(1);
6975
// Budget for probing URLs recovered from the backend's persisted runtime
7076
// state (server-runtime.json) after every static candidate timed out.
@@ -345,6 +351,11 @@ interface BackendManagerState {
345351
// Consecutive readiness timeouts (process alive, no candidate URL
346352
// answering), reset when a run reaches ready or an external stop() runs.
347353
readonly readinessFailureAttempt: number;
354+
// Consecutive runs that spawned but exited before ever becoming ready.
355+
// Reset when a run reaches ready, on external stop(), and on a fresh
356+
// manual start — so the cap firing once doesn't permanently remove the
357+
// retry budget.
358+
readonly exitFailureAttempt: number;
348359
readonly restartFiber: Option.Option<Fiber.Fiber<void, never>>;
349360
readonly nextRunId: number;
350361
}
@@ -357,6 +368,7 @@ const initialState: BackendManagerState = {
357368
restartAttempt: 0,
358369
preflightFailureAttempt: 0,
359370
readinessFailureAttempt: 0,
371+
exitFailureAttempt: 0,
360372
restartFiber: Option.none(),
361373
nextRunId: 1,
362374
};
@@ -799,6 +811,9 @@ export const makeBackendInstance = Effect.fn("makeBackendInstance")(function* (
799811
ready: false,
800812
config: Option.some(config.value),
801813
preflightFailureAttempt: resetFatalPreflightCounter ? 0 : latest.preflightFailureAttempt,
814+
// A fresh manual start gets a fresh never-ready exit budget; a
815+
// restart-loop start (desiredRunning already set) keeps the streak.
816+
exitFailureAttempt: current.desiredRunning ? latest.exitFailureAttempt : 0,
802817
}));
803818

804819
const preflightFailure = config.value.preflightFailure;
@@ -895,55 +910,80 @@ export const makeBackendInstance = Effect.fn("makeBackendInstance")(function* (
895910
) {
896911
yield* mutex.withPermits(1)(
897912
Effect.gen(function* () {
898-
const { isCurrentRun, nextState, pid, exitObserved, stopRequested, wasReady } =
899-
yield* Ref.modify(
900-
state,
901-
(
902-
latest,
903-
): readonly [
904-
{
905-
readonly isCurrentRun: boolean;
906-
readonly nextState: BackendManagerState;
907-
readonly pid: Option.Option<number>;
908-
readonly exitObserved: boolean;
909-
readonly stopRequested: boolean;
910-
readonly wasReady: boolean;
911-
},
912-
BackendManagerState,
913-
] => {
914-
const currentRun = Option.getOrUndefined(latest.active);
915-
if (currentRun?.id !== runId) {
916-
return [
917-
{
918-
isCurrentRun: false,
919-
nextState: latest,
920-
pid: Option.none<number>(),
921-
exitObserved: false,
922-
stopRequested: false,
923-
wasReady: false,
924-
},
925-
latest,
926-
] as const;
927-
}
928-
929-
const next = {
930-
...latest,
931-
active: Option.none<ActiveBackendRun>(),
932-
ready: false,
933-
};
913+
const {
914+
isCurrentRun,
915+
nextState,
916+
pid,
917+
exitObserved,
918+
stopRequested,
919+
wasReady,
920+
exitFailureAttempt,
921+
} = yield* Ref.modify(
922+
state,
923+
(
924+
latest,
925+
): readonly [
926+
{
927+
readonly isCurrentRun: boolean;
928+
readonly nextState: BackendManagerState;
929+
readonly pid: Option.Option<number>;
930+
readonly exitObserved: boolean;
931+
readonly stopRequested: boolean;
932+
readonly wasReady: boolean;
933+
// Some(n) when this exit counted toward the
934+
// never-became-ready streak.
935+
readonly exitFailureAttempt: Option.Option<number>;
936+
},
937+
BackendManagerState,
938+
] => {
939+
const currentRun = Option.getOrUndefined(latest.active);
940+
if (currentRun?.id !== runId) {
934941
return [
935942
{
936-
isCurrentRun: true,
937-
nextState: next,
938-
pid: currentRun.pid,
939-
exitObserved: currentRun.exitObserved,
940-
stopRequested: currentRun.stopRequested,
941-
wasReady: latest.ready,
943+
isCurrentRun: false,
944+
nextState: latest,
945+
pid: Option.none<number>(),
946+
exitObserved: false,
947+
stopRequested: false,
948+
wasReady: false,
949+
exitFailureAttempt: Option.none<number>(),
942950
},
943-
next,
951+
latest,
944952
] as const;
945-
},
946-
);
953+
}
954+
955+
const countsTowardExitCap =
956+
currentRun.exitObserved &&
957+
!currentRun.stopRequested &&
958+
!latest.ready &&
959+
latest.desiredRunning;
960+
const nextExitFailureAttempt = latest.ready
961+
? 0
962+
: countsTowardExitCap
963+
? latest.exitFailureAttempt + 1
964+
: latest.exitFailureAttempt;
965+
const next = {
966+
...latest,
967+
active: Option.none<ActiveBackendRun>(),
968+
ready: false,
969+
exitFailureAttempt: nextExitFailureAttempt,
970+
};
971+
return [
972+
{
973+
isCurrentRun: true,
974+
nextState: next,
975+
pid: currentRun.pid,
976+
exitObserved: currentRun.exitObserved,
977+
stopRequested: currentRun.stopRequested,
978+
wasReady: latest.ready,
979+
exitFailureAttempt: countsTowardExitCap
980+
? Option.some(nextExitFailureAttempt)
981+
: Option.none<number>(),
982+
},
983+
next,
984+
] as const;
985+
},
986+
);
947987

948988
if (isCurrentRun) {
949989
yield* desktopTelemetryPublisher.removeControlSource(spec.id);
@@ -962,6 +1002,36 @@ export const makeBackendInstance = Effect.fn("makeBackendInstance")(function* (
9621002
}
9631003

9641004
if (isCurrentRun && nextState.desiredRunning) {
1005+
if (
1006+
Option.isSome(exitFailureAttempt) &&
1007+
exitFailureAttempt.value >= MAX_EXIT_FAILURE_ATTEMPTS
1008+
) {
1009+
// The backend spawns fine but keeps dying before it ever
1010+
// answers a readiness probe. Silent restarts can't fix
1011+
// that — surface it exactly like an exhausted preflight
1012+
// (dialog + Windows fallback on the primary, inline error
1013+
// on the WSL secondary) instead of looping forever.
1014+
yield* logInstanceError(
1015+
"backend keeps exiting before becoming ready; surfacing",
1016+
{ reason, attempt: exitFailureAttempt.value },
1017+
);
1018+
const shouldRestart = yield* (
1019+
spec.onPreflightFailed?.({
1020+
reason: `The backend exited ${exitFailureAttempt.value} times in a row before becoming ready (last exit: ${reason}).`,
1021+
fatal: false,
1022+
retryLimit: MAX_EXIT_FAILURE_ATTEMPTS,
1023+
}) ?? Effect.succeed(false)
1024+
);
1025+
yield* Ref.update(state, (latest) => ({
1026+
...latest,
1027+
exitFailureAttempt: 0,
1028+
...(shouldRestart ? {} : { desiredRunning: false }),
1029+
}));
1030+
if (shouldRestart) {
1031+
yield* scheduleRestart(reason);
1032+
}
1033+
return;
1034+
}
9651035
yield* scheduleRestart(reason);
9661036
}
9671037
}),
@@ -1000,6 +1070,7 @@ export const makeBackendInstance = Effect.fn("makeBackendInstance")(function* (
10001070
...latest,
10011071
restartAttempt: 0,
10021072
readinessFailureAttempt: 0,
1073+
exitFailureAttempt: 0,
10031074
ready: true,
10041075
// Rebind the advertised base URL to the candidate that
10051076
// actually answered, so consumers that read currentConfig
@@ -1187,6 +1258,7 @@ export const makeBackendInstance = Effect.fn("makeBackendInstance")(function* (
11871258
desiredRunning: false,
11881259
ready: false,
11891260
readinessFailureAttempt: 0,
1261+
exitFailureAttempt: 0,
11901262
active,
11911263
restartFiber: Option.none<Fiber.Fiber<void, never>>(),
11921264
},

0 commit comments

Comments
 (0)