Skip to content

Commit 36caf34

Browse files
t3dotggclaude
andauthored
fix(desktop,web): contain renderer memory growth and recover from renderer OOM crashes (pingdotgg#5148)
Co-authored-by: Claude Fable 5 <[email protected]>
1 parent 90e3778 commit 36caf34

4 files changed

Lines changed: 173 additions & 8 deletions

File tree

‎apps/desktop/src/window/DesktopWindow.ts‎

Lines changed: 41 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import * as Clock from "effect/Clock";
12
import * as Context from "effect/Context";
23
import * as Effect from "effect/Effect";
34
import * as Fiber from "effect/Fiber";
@@ -25,6 +26,12 @@ const TITLEBAR_LIGHT_SYMBOL_COLOR = "#1f2937";
2526
const TITLEBAR_DARK_SYMBOL_COLOR = "#f8fafc";
2627
const MAIN_WINDOW_BOUNDS_PERSIST_DEBOUNCE_MS = 500;
2728
const DEVELOPMENT_LOAD_RETRY_DELAYS_MS = [100, 250, 500, 1_000, 2_000] as const;
29+
// Renderer crash (usually V8 OOM on long sessions) recovery: reload after a
30+
// short delay, at most MAX_ATTEMPTS times per rolling WINDOW so a renderer
31+
// that dies on boot cannot reload-loop forever.
32+
const RENDERER_RECOVERY_RELOAD_DELAY_MS = 500;
33+
const RENDERER_RECOVERY_MAX_ATTEMPTS = 3;
34+
const RENDERER_RECOVERY_WINDOW_MS = 60_000;
2835
const DEVELOPMENT_RETRYABLE_LOAD_ERROR_CODES = new Set([
2936
-2, // ERR_FAILED
3037
-7, // ERR_TIMED_OUT
@@ -549,6 +556,7 @@ export const make = Effect.gen(function* () {
549556

550557
let developmentLoadRetryIndex = 0;
551558
let developmentLoadRetryFiber: Fiber.Fiber<void, never> | undefined;
559+
let rendererRecoveryTimestamps: number[] = [];
552560
const clearDevelopmentLoadRetry = () => {
553561
if (developmentLoadRetryFiber === undefined) {
554562
return;
@@ -630,10 +638,39 @@ export const make = Effect.gen(function* () {
630638
},
631639
);
632640
window.webContents.on("render-process-gone", (_event, details) => {
633-
void runPromise(
634-
logWindowWarning("main window render process gone", {
635-
reason: details.reason,
636-
exitCode: details.exitCode,
641+
const recoverable =
642+
details.reason === "crashed" ||
643+
details.reason === "oom" ||
644+
details.reason === "abnormal-exit";
645+
// Long sessions can OOM the renderer (V8 heap exhaustion from
646+
// accumulated thread state). Without a reload the user is left staring
647+
// at a dead white window while agents keep running invisibly, so
648+
// recover by reloading — the renderer rehydrates from the backend,
649+
// which is unaffected. Recovery attempts are bounded so a renderer
650+
// that dies immediately on boot cannot reload-loop forever.
651+
runFork(
652+
Effect.gen(function* () {
653+
const now = yield* Clock.currentTimeMillis;
654+
rendererRecoveryTimestamps = rendererRecoveryTimestamps.filter(
655+
(timestamp) => now - timestamp < RENDERER_RECOVERY_WINDOW_MS,
656+
);
657+
const shouldRecover =
658+
recoverable &&
659+
!window.isDestroyed() &&
660+
rendererRecoveryTimestamps.length < RENDERER_RECOVERY_MAX_ATTEMPTS;
661+
yield* logWindowWarning("main window render process gone", {
662+
reason: details.reason,
663+
exitCode: details.exitCode,
664+
recovering: shouldRecover,
665+
});
666+
if (!shouldRecover) {
667+
return;
668+
}
669+
rendererRecoveryTimestamps.push(now);
670+
yield* Effect.sleep(RENDERER_RECOVERY_RELOAD_DELAY_MS);
671+
if (!window.isDestroyed()) {
672+
loadApplication();
673+
}
637674
}),
638675
);
639676
});

‎apps/web/src/components/Sidebar.logic.ts‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,12 @@ import { resolveServerBackedAppStageLabel } from "../branding.logic";
1616
export const THREAD_SELECTION_SAFE_SELECTOR = "[data-thread-item], [data-thread-selection-safe]";
1717
export const THREAD_JUMP_HINT_SHOW_DELAY_MS = 100;
1818
// Visible sidebar rows are prewarmed into the thread-detail cache so opening a
19-
// nearby thread usually reuses an already-hot subscription.
20-
export const SIDEBAR_THREAD_PREWARM_LIMIT = 10;
19+
// nearby thread usually reuses an already-hot subscription. Each prewarmed
20+
// thread holds a live, fully hydrated detail subscription (all messages and
21+
// activities, growing as agents work) for as long as the row stays visible,
22+
// so this limit is a direct renderer-heap and server-load multiplier — keep
23+
// it small; cold opens still render instantly from the cached snapshot.
24+
export const SIDEBAR_THREAD_PREWARM_LIMIT = 3;
2125

2226
type SidebarProject = {
2327
id: string;

‎packages/client-runtime/src/state/threadReducer.test.ts‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -633,6 +633,94 @@ describe("applyThreadDetailEvent", () => {
633633
expect(result.thread.activities[0]?.id).toBe("activity-0");
634634
}
635635
});
636+
637+
it("replaces earlier resolvable context-window updates for the same turn", () => {
638+
const contextWindowActivity = (id: string, sequence: number, usedTokens: unknown) => ({
639+
id: EventId.make(id),
640+
tone: "info" as const,
641+
kind: "context-window.updated",
642+
summary: "Context window updated",
643+
payload: { usedTokens },
644+
turnId: TurnId.make("turn-1"),
645+
sequence,
646+
createdAt: "2026-04-01T11:00:00.000Z",
647+
});
648+
const otherTurnActivity = contextWindowActivity("activity-other-turn", 2, 500);
649+
const existingActivities = [
650+
contextWindowActivity("activity-cw-1", 1, 1_000),
651+
{ ...otherTurnActivity, turnId: TurnId.make("turn-0") },
652+
// Malformed row (no usedTokens): must survive, and must not be
653+
// treated as the latest value by consumers.
654+
contextWindowActivity("activity-cw-malformed", 3, undefined),
655+
contextWindowActivity("activity-cw-2", 4, 2_000),
656+
];
657+
658+
const result = applyThreadDetailEvent(
659+
{ ...baseThread, activities: existingActivities },
660+
{
661+
...baseEventFields,
662+
sequence: 20,
663+
occurredAt: "2026-04-01T11:02:00.000Z",
664+
aggregateKind: "thread",
665+
aggregateId: ThreadId.make("thread-1"),
666+
type: "thread.activity-appended",
667+
payload: {
668+
threadId: ThreadId.make("thread-1"),
669+
activity: contextWindowActivity("activity-cw-3", 5, 3_000),
670+
},
671+
},
672+
);
673+
674+
expect(result.kind).toBe("updated");
675+
if (result.kind === "updated") {
676+
const ids = result.thread.activities.map((activity) => activity.id);
677+
// Same-turn resolvable rows collapse to the newest; the other turn's
678+
// row and the malformed row are untouched.
679+
expect(ids).toEqual(["activity-other-turn", "activity-cw-malformed", "activity-cw-3"]);
680+
}
681+
});
682+
683+
it("does not collapse context-window history for a malformed update", () => {
684+
const resolvable = {
685+
id: EventId.make("activity-cw-resolvable"),
686+
tone: "info" as const,
687+
kind: "context-window.updated",
688+
summary: "Context window updated",
689+
payload: { usedTokens: 1_000 },
690+
turnId: TurnId.make("turn-1"),
691+
sequence: 1,
692+
createdAt: "2026-04-01T11:00:00.000Z",
693+
};
694+
695+
const result = applyThreadDetailEvent(
696+
{ ...baseThread, activities: [resolvable] },
697+
{
698+
...baseEventFields,
699+
sequence: 21,
700+
occurredAt: "2026-04-01T11:03:00.000Z",
701+
aggregateKind: "thread",
702+
aggregateId: ThreadId.make("thread-1"),
703+
type: "thread.activity-appended",
704+
payload: {
705+
threadId: ThreadId.make("thread-1"),
706+
activity: {
707+
...resolvable,
708+
id: EventId.make("activity-cw-broken"),
709+
payload: { usedTokens: Number.NaN },
710+
sequence: 2,
711+
},
712+
},
713+
},
714+
);
715+
716+
expect(result.kind).toBe("updated");
717+
if (result.kind === "updated") {
718+
// The resolvable row must survive so consumers can still derive a
719+
// usage value by walking backwards past the malformed row.
720+
const ids = result.thread.activities.map((activity) => activity.id);
721+
expect(ids).toEqual(["activity-cw-resolvable", "activity-cw-broken"]);
722+
}
723+
});
636724
});
637725

638726
describe("thread.turn-diff-completed", () => {

‎packages/client-runtime/src/state/threadReducer.ts‎

Lines changed: 38 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,24 @@ const activityOrder = O.combineAll<OrchestrationThreadActivity>([
3535
O.mapInput(O.String, (a) => a.id),
3636
]);
3737

38+
/**
39+
* Matches the validity rule in `deriveLatestContextWindowSnapshot` (and the
40+
* server's snapshot-side `dropStaleContextWindowActivities`): rows without a
41+
* finite, non-negative `usedTokens` are skipped during the consumer's backward
42+
* walk, so they must not replace an earlier resolvable row here.
43+
*/
44+
function isResolvableContextWindowActivity(activity: OrchestrationThreadActivity): boolean {
45+
if (activity.kind !== "context-window.updated") {
46+
return false;
47+
}
48+
const payload =
49+
activity.payload && typeof activity.payload === "object"
50+
? (activity.payload as Record<string, unknown>)
51+
: null;
52+
const usedTokens = payload?.usedTokens;
53+
return typeof usedTokens === "number" && Number.isFinite(usedTokens) && usedTokens >= 0;
54+
}
55+
3856
/**
3957
* Apply a single orchestration event to an `OrchestrationThread`, returning
4058
* the updated thread, a deletion signal, or an "unchanged" marker when the
@@ -509,10 +527,28 @@ export function applyThreadDetailEvent(
509527

510528
// ── Activities ──────────────────────────────────────────────────
511529
case "thread.activity-appended": {
530+
const activity = event.payload.activity;
531+
// A resolvable context-window update supersedes earlier resolvable ones
532+
// for the same turn: consumers only read the latest value (walking the
533+
// array backwards), and providers stream these updates continuously, so
534+
// retaining the history grows the thread by thousands of rows over a
535+
// long session. Mirrors the server-side snapshot rule in
536+
// dropStaleContextWindowActivities; retention stays per turn so a
537+
// thread.reverted that discards turns can still resolve a value from
538+
// the turns that survive.
539+
const supersedesContextWindow = isResolvableContextWindowActivity(activity);
512540
const activities = pipe(
513541
thread.activities,
514-
Arr.filter((activity) => activity.id !== event.payload.activity.id),
515-
Arr.append(event.payload.activity),
542+
Arr.filter(
543+
(entry) =>
544+
entry.id !== activity.id &&
545+
!(
546+
supersedesContextWindow &&
547+
entry.turnId === activity.turnId &&
548+
isResolvableContextWindowActivity(entry)
549+
),
550+
),
551+
Arr.append(activity),
516552
Arr.sort(activityOrder),
517553
);
518554

0 commit comments

Comments
 (0)