Skip to content

Commit 969631c

Browse files
committed
perf(web): release left threads past a byte budget
Threads the user navigated to and left may keep their state up to a byte budget, the largest released first. A released thread, by the budget or the idle TTL, keeps the end of its timeline for an immediate paint. Other mounts, unfinished runs, pending questions and approvals, unsaved threads and open threads stay outside the budget, and mobile sets no budget. A saved record that #26435 discards for a missing question card does not count as a copy to come back from. A thread the idle TTL released reopens from its record with the question still answerable.
1 parent 06c22d7 commit 969631c

9 files changed

Lines changed: 930 additions & 20 deletions

File tree

‎apps/web/src/components/ChatView.logic.test.ts‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ import * as DateTime from "effect/DateTime";
3232
import * as Option from "effect/Option";
3333
import { afterEach, describe, expect, it, vi } from "vite-plus/test";
3434
import { Atom } from "effect/reactivity";
35+
import { rememberTimelinePosition } from "./chat/timelineScrollAnchoring";
3536
import { appAtomRegistry } from "../rpc/atomRegistry";
3637
import { environmentThreadDetails } from "../state/threads";
3738

@@ -42,6 +43,7 @@ import {
4243
resolveComposerProviderSelection,
4344
resolveWorktreeSetupProgress,
4445
peekRememberedThreadTimeline,
46+
releaseRememberedThreadTimeline,
4547
rememberReadyThreadTimeline,
4648
resetHeldThreadTimeline,
4749
resolveThreadSwitchTimeline,
@@ -423,6 +425,48 @@ describe("resolveThreadSwitchTimeline", () => {
423425
).toEqual(["a1", "a2"]);
424426
});
425427

428+
describe("a released thread", () => {
429+
const entries = Array.from({ length: 200 }, (_, index) => `entry-${index}`);
430+
const threadKey = "env-1:released";
431+
432+
it("keeps only the end of its remembered timeline", () => {
433+
rememberReadyThreadTimeline({ threadKey, entries });
434+
releaseRememberedThreadTimeline(threadKey);
435+
const kept = peekRememberedThreadTimeline<string[]>(threadKey)!;
436+
expect(kept.length).toBeLessThan(100);
437+
expect(kept).toEqual(entries.slice(-kept.length));
438+
expect(
439+
resolveThreadSwitchTimeline({ loading: true, activeThreadKey: threadKey, nextEntries: [] }),
440+
).toEqual(kept);
441+
});
442+
443+
it("leaves other threads' remembered timelines whole", () => {
444+
rememberReadyThreadTimeline({ threadKey: "env-1:other", entries });
445+
rememberReadyThreadTimeline({ threadKey, entries });
446+
releaseRememberedThreadTimeline(threadKey);
447+
expect(peekRememberedThreadTimeline<string[]>("env-1:other")).toEqual(entries);
448+
});
449+
450+
it("keeps none when the reader's place is further up the timeline", () => {
451+
rememberReadyThreadTimeline({ threadKey, entries });
452+
rememberTimelinePosition(
453+
threadKey,
454+
{ rowId: "entry-3", offsetWithinRow: 0, scrollOffset: 0, atEnd: false },
455+
false,
456+
);
457+
releaseRememberedThreadTimeline(threadKey);
458+
expect(peekRememberedThreadTimeline(threadKey)).toBeNull();
459+
});
460+
461+
it("still remembers sixteen threads", () => {
462+
const keys = Array.from({ length: 18 }, (_, index) => `env-1:thread-${index}`);
463+
for (const key of keys) rememberReadyThreadTimeline({ threadKey: key, entries: [key] });
464+
expect(keys.map((key) => peekRememberedThreadTimeline(key) !== null)).toEqual(
465+
keys.map((_, index) => index >= 2),
466+
);
467+
});
468+
});
469+
426470
it("does not paint the last-viewed thread for a destination with nothing remembered", () => {
427471
rememberReadyThreadTimeline(held);
428472
expect(

‎apps/web/src/components/ChatView.logic.ts‎

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,12 @@ import {
7676
} from "../composerDraftStore";
7777
import * as Schema from "effect/Schema";
7878
import { appAtomRegistry } from "../rpc/atomRegistry";
79-
import { environmentThreadShells, environmentThreadDetails } from "../state/threads";
79+
import {
80+
environmentThreadDetails,
81+
environmentThreadShells,
82+
onLeftThreadReleased,
83+
} from "../state/threads";
84+
import { readTimelinePosition } from "./chat/timelineScrollAnchoring";
8085
import { waitForAtomValue } from "../state/waitForAtomValue";
8186
import {
8287
filterTerminalContextsWithText,
@@ -383,6 +388,31 @@ export function peekRememberedThreadTimeline<T extends readonly unknown[]>(
383388
return (rememberedThreadTimelines.get(threadKey)?.entries as T | undefined) ?? null;
384389
}
385390

391+
// Fills the pane at the end of a timeline until the thread is back from the cache.
392+
const RELEASED_THREAD_TIMELINE_ENTRIES = 60;
393+
394+
/**
395+
* These rows point at the thread's items, so a released thread keeps only its last few. A reader's
396+
* place further up can only be restored on the whole timeline, so that thread keeps none.
397+
*/
398+
export function releaseRememberedThreadTimeline(threadKey: string): void {
399+
const held = rememberedThreadTimelines.get(threadKey);
400+
if (held === undefined) return;
401+
if (readTimelinePosition(threadKey)?.atEnd === false) {
402+
rememberedThreadTimelines.delete(threadKey);
403+
rememberedThreadTimelineOrder = rememberedThreadTimelineOrder.filter(
404+
(key) => key !== threadKey,
405+
);
406+
return;
407+
}
408+
rememberedThreadTimelines.set(threadKey, {
409+
threadKey,
410+
entries: held.entries.slice(-RELEASED_THREAD_TIMELINE_ENTRIES),
411+
});
412+
}
413+
414+
onLeftThreadReleased(releaseRememberedThreadTimeline);
415+
386416
export function resetHeldThreadTimeline(): void {
387417
rememberedThreadTimelines = new Map();
388418
rememberedThreadTimelineOrder = [];

‎apps/web/src/components/chat/ThreadTimeline.tsx‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,16 @@
11
import type { ScopedThreadRef } from "@t3tools/contracts";
2-
import { type ComponentProps, useLayoutEffect } from "react";
2+
import { type ComponentProps, use, useLayoutEffect } from "react";
33
import { useIncrementalTimelineEntries } from "../../hooks/useIncrementalTimelineEntries";
44
import type { TimelineEntriesInput, TimelineEntry } from "../../session-logic";
55
import { useTimelineVisibleTurnItems } from "../../state/entities";
6+
import { markThreadVisited } from "../../state/threads";
67
import {
78
peekRememberedThreadTimeline,
89
rememberReadyThreadTimeline,
910
resolveThreadSwitchTimeline,
1011
timelineHasEphemeralPreviewUrls,
1112
} from "../ChatView.logic";
13+
import { BackgroundThreadContext } from "./backgroundThread";
1214
import { MessagesTimeline } from "./MessagesTimeline";
1315

1416
/**
@@ -47,6 +49,12 @@ export function ThreadTimeline({
4749
nextEntries: entries,
4850
rememberedForActive: peekRememberedThreadTimeline<typeof entries>(activeThreadKey),
4951
});
52+
// A side chat shows its thread beside the one the user navigated to.
53+
const backgroundThread = use(BackgroundThreadContext);
54+
// oxlint-disable-next-line t3code/no-unreasoned-state-sync -- tells the left-thread budget which thread the user navigated to, once it is on screen
55+
useLayoutEffect(() => {
56+
if (!backgroundThread && threadDetailRef !== null) markThreadVisited(threadDetailRef);
57+
}, [backgroundThread, threadDetailRef]);
5058
// oxlint-disable-next-line t3code/no-unreasoned-state-sync -- stores the committed ready timeline in the module-level cache used during thread switches
5159
useLayoutEffect(() => {
5260
if (detailLoading || entries.length === 0 || timelineHasEphemeralPreviewUrls(entries)) {

‎apps/web/src/state/threads.ts‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { createChildThreadApprovalsAtom } from "@t3tools/client-runtime/state/thread-requests";
22
import { useAtomValue } from "@effect/atom-react";
33
import { enabledEnvironmentIds } from "@t3tools/client-runtime/state/connections";
4+
import { scopedThreadKey } from "@t3tools/client-runtime/environment";
45
import { arrayElementsEqual } from "@t3tools/client-runtime/state/entities";
56
import {
67
createEnvironmentThreadDetailAtoms,
@@ -33,7 +34,24 @@ export const threadEnvironment = createThreadEnvironmentAtoms(
3334
connectionAtomRuntime,
3435
threadFamilies.snapshotAtom,
3536
);
36-
const environmentThreads = createEnvironmentThreadStateAtoms(connectionAtomRuntime);
37+
// What left threads may keep in memory. A thread opened from a bounded server window costs a few MB
38+
// at most, so only threads grown by long streaming or scrollback reach it.
39+
const LEFT_THREADS_MAX_BYTES = 50 * 1024 * 1024;
40+
41+
let leftThreadReleased: (threadKey: string) => void = () => {};
42+
43+
/** Whoever remembers more of a left thread lets go of it here, together with its state. */
44+
export function onLeftThreadReleased(listener: (threadKey: string) => void): void {
45+
leftThreadReleased = listener;
46+
}
47+
48+
const environmentThreads = createEnvironmentThreadStateAtoms(connectionAtomRuntime, {
49+
leftThreads: {
50+
maxBytes: LEFT_THREADS_MAX_BYTES,
51+
onRelease: (thread) => leftThreadReleased(scopedThreadKey(thread)),
52+
},
53+
});
54+
export const markThreadVisited = environmentThreads.markThreadVisited;
3755
export const environmentThreadDetails = createEnvironmentThreadDetailAtoms(
3856
environmentThreads.stateAtom,
3957
);
Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
import type { OrchestrationV2ThreadProjection } from "@t3tools/contracts";
2+
import { describe, expect, it } from "vite-plus/test";
3+
4+
import { v2Projection } from "./orchestrationV2TestFixtures.ts";
5+
import { estimateRetainedThreadBytes } from "./threadRetention.ts";
6+
7+
const withItems = (count: number, bytes: number): OrchestrationV2ThreadProjection => {
8+
const items = Array.from({ length: count }, (_, index) => ({
9+
id: `item-${index}`,
10+
threadId: v2Projection.thread.id,
11+
type: "system_notice",
12+
message: "x".repeat(bytes),
13+
}));
14+
return {
15+
...v2Projection,
16+
turnItems: items,
17+
visibleTurnItems: items.map((item, position) => ({
18+
position,
19+
visibility: "local",
20+
sourceThreadId: v2Projection.thread.id,
21+
sourceItemId: item.id,
22+
item,
23+
})),
24+
} as unknown as OrchestrationV2ThreadProjection;
25+
};
26+
27+
describe("estimateRetainedThreadBytes", () => {
28+
it("counts text once for an item the timeline shares, and grows with the items", () => {
29+
const small = estimateRetainedThreadBytes(withItems(10, 100));
30+
const large = estimateRetainedThreadBytes(withItems(10, 100_000));
31+
expect(large - small).toBe(10 * (100_000 - 100));
32+
expect(estimateRetainedThreadBytes(withItems(20, 100))).toBeGreaterThan(small * 1.9);
33+
});
34+
35+
it("charges an inherited row, which no local item shares", () => {
36+
const local = withItems(1, 1_000);
37+
const inherited = {
38+
...local,
39+
turnItems: [],
40+
visibleTurnItems: local.visibleTurnItems.map((row) => ({
41+
...row,
42+
visibility: "inherited",
43+
sourceThreadId: "parent",
44+
})),
45+
} as unknown as OrchestrationV2ThreadProjection;
46+
expect(estimateRetainedThreadBytes(inherited)).toBeGreaterThan(1_000);
47+
});
48+
});
Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1 +1,42 @@
1+
import type { OrchestrationV2RunStatus, OrchestrationV2ThreadProjection } from "@t3tools/contracts";
2+
13
export const THREAD_SNAPSHOT_IDLE_TTL_MS = 5 * 60_000;
4+
5+
/**
6+
* Runs the user is likely to come back to. Preparing, starting and running ones also have newer
7+
* state than the persisted cache, which is saved only once they settle.
8+
*/
9+
export const UNFINISHED_RUN_STATUSES: ReadonlySet<OrchestrationV2RunStatus> = new Set([
10+
"preparing",
11+
"queued",
12+
"starting",
13+
"running",
14+
"waiting",
15+
]);
16+
17+
// What the heap charges a small object beyond its strings.
18+
const OBJECT_OVERHEAD_BYTES = 64;
19+
20+
function retainedBytes(value: unknown): number {
21+
if (typeof value === "string") return value.length;
22+
if (typeof value !== "object" || value === null) return 8;
23+
let bytes = OBJECT_OVERHEAD_BYTES;
24+
for (const field of Object.values(value)) bytes += retainedBytes(field);
25+
return bytes;
26+
}
27+
28+
/**
29+
* A cheap estimate of what a thread keeps in the heap, from its items, inherited rows and message
30+
* records, with strings counted by length. It walks one thread when the thread is left. Timeline
31+
* rows and parsed markdown come on top, which matters only for small threads.
32+
*/
33+
export function estimateRetainedThreadBytes(projection: OrchestrationV2ThreadProjection): number {
34+
let bytes = retainedBytes(projection.turnItems) + retainedBytes(projection.messages);
35+
for (const row of projection.visibleTurnItems) {
36+
bytes +=
37+
row.sourceThreadId === projection.thread.id
38+
? OBJECT_OVERHEAD_BYTES
39+
: retainedBytes(row.item) + OBJECT_OVERHEAD_BYTES;
40+
}
41+
return bytes;
42+
}

0 commit comments

Comments
 (0)