Skip to content

Commit 2f24391

Browse files
committed
fix(history): enforce page budgets inside human turns and discard oversized caches
1 parent 66ee628 commit 2f24391

7 files changed

Lines changed: 231 additions & 49 deletions

File tree

‎apps/server/src/orchestration-v2/ProjectionStore.test.ts‎

Lines changed: 49 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ import * as ProjectionStore from "./ProjectionStore.ts";
4040
import {
4141
buildBoundedThreadProjection,
4242
decodeThreadHistoryCursor,
43+
encodeThreadHistoryCursor,
4344
selectHistoryPageFromCursor,
4445
THREAD_HISTORY_PAGE_POLICY,
4546
} from "./threadHistoryPaging.ts";
@@ -1529,6 +1530,52 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => {
15291530
[baselineId],
15301531
);
15311532
}
1533+
const humanWindow = yield* projectionStore.getThreadSnapshotWindow(threadId, {
1534+
rowLimit: 77,
1535+
userTurnLimit: THREAD_HISTORY_PAGE_POLICY.maxUserTurns,
1536+
});
1537+
assert.lengthOf(humanWindow.projection.turnItems, 77);
1538+
1539+
// Fewer than 75 rows still need a byte pass after materializing wire payloads.
1540+
yield* sql`
1541+
UPDATE orchestration_v2_projection_turn_items
1542+
SET type = 'assistant_message', payload_json = json_set(payload_json,
1543+
'$.type', 'assistant_message', '$.text', ${"x".repeat(40_000)},
1544+
'$.messageId', turn_item_id, '$.streaming', json('false'))
1545+
WHERE thread_id = ${threadId} AND ordinal BETWEEN 2 AND 70
1546+
`;
1547+
const byteCursor = encodeThreadHistoryCursor({
1548+
snapshotSequence: 0,
1549+
sourceThreadId: threadId,
1550+
sourceItemId: TurnItemId.make("turn-item:bounded-sql-history:71"),
1551+
position: 70,
1552+
});
1553+
const bytePage = yield* projectionStore.getThreadHistoryPage(threadId, byteCursor);
1554+
assert.isAbove(bytePage.items.length, 0);
1555+
assert.isBelow(bytePage.items.length, 70);
1556+
assert.isAtMost(
1557+
Buffer.byteLength(JSON.stringify(bytePage.items), "utf8"),
1558+
THREAD_HISTORY_PAGE_POLICY.maxEncodedBytes,
1559+
);
1560+
const byteIds = bytePage.items.map((row) => String(row.sourceItemId));
1561+
let byteNext = bytePage.nextCursor;
1562+
for (let pages = 0; byteNext !== null; pages += 1) {
1563+
assert.isBelow(pages, 70);
1564+
const page = yield* projectionStore.getThreadHistoryPage(threadId, byteNext);
1565+
assert.isAbove(page.items.length, 0);
1566+
byteIds.unshift(...page.items.map((row) => String(row.sourceItemId)));
1567+
byteNext = page.nextCursor;
1568+
}
1569+
assert.deepEqual(
1570+
byteIds,
1571+
Array.from({ length: 70 }, (_, index) => `turn-item:bounded-sql-history:${index + 1}`),
1572+
);
1573+
yield* sql`
1574+
UPDATE orchestration_v2_projection_turn_items
1575+
SET type = 'command_execution', payload_json = json_set(payload_json, '$.type', 'command_execution')
1576+
WHERE thread_id = ${threadId} AND ordinal BETWEEN 2 AND 70
1577+
`;
1578+
15321579
const retainedRequestPlan = yield* sql<{ readonly detail: string }>`
15331580
EXPLAIN QUERY PLAN
15341581
WITH selected AS (
@@ -1571,9 +1618,9 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => {
15711618
const sqlPageLimit = THREAD_HISTORY_PAGE_POLICY.maxItems + 2;
15721619
const initialSnapshot = yield* projectionStore.getThreadSnapshotWindow(threadId, {
15731620
rowLimit: sqlPageLimit,
1621+
userTurnLimit: THREAD_HISTORY_PAGE_POLICY.maxUserTurns,
15741622
});
15751623
const initialPage = buildBoundedThreadProjection({
1576-
policy: { ...THREAD_HISTORY_PAGE_POLICY, maxUserTurns: undefined },
15771624
projection: initialSnapshot.projection,
15781625
snapshotSequence: initialSnapshot.snapshotSequence,
15791626
});
@@ -1586,10 +1633,10 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => {
15861633
const anchorItemId = TurnItemId.make(decodeThreadHistoryCursor(cursor).si);
15871634
const snapshot = yield* projectionStore.getThreadSnapshotWindow(threadId, {
15881635
rowLimit: sqlPageLimit,
1636+
userTurnLimit: 20,
15891637
anchorItemId,
15901638
});
15911639
const page = selectHistoryPageFromCursor({
1592-
policy: { ...THREAD_HISTORY_PAGE_POLICY, maxUserTurns: undefined },
15931640
items: snapshot.projection.visibleTurnItems,
15941641
cursor,
15951642
snapshotSequence: snapshot.snapshotSequence,

‎apps/server/src/orchestration-v2/ProjectionStore.ts‎

Lines changed: 4 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -2963,11 +2963,7 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
29632963
FROM eligible
29642964
WHERE ordinal >= (SELECT ordinal FROM boundary)
29652965
ORDER BY ordinal DESC, turn_item_id DESC
2966-
LIMIT CASE
2967-
WHEN ${window.rowLimit} = 0 THEN 0
2968-
WHEN (SELECT COUNT(*) FROM user_anchors) > 0 THEN -1
2969-
ELSE ${window.rowLimit}
2970-
END
2966+
LIMIT ${window.rowLimit}
29712967
), retained AS MATERIALIZED (
29722968
SELECT item_rowid, ordinal, turn_item_id FROM selected
29732969
UNION
@@ -5496,12 +5492,6 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
54965492
};
54975493
}),
54985494
);
5499-
if (hasUserTurns)
5500-
return {
5501-
...page,
5502-
snapshotSequence,
5503-
items: items.map((row, position) => ({ ...row, position })),
5504-
};
55055495
const bounded = selectRecentTimelineWindow({
55065496
items,
55075497
snapshotSequence,
@@ -7382,7 +7372,9 @@ export const layerMemory: Layer.Layer<ProjectionStoreV2> = Layer.effect(
73827372
? rawStart
73837373
: anchors.at(-anchorLimit)!
73847374
: Math.max(0, anchorIndex - options.rowLimit);
7385-
const visibleTurnItems = candidates.slice(start);
7375+
const visibleTurnItems = candidates.slice(
7376+
Math.max(start, anchorIndex - Math.max(0, options.rowLimit)),
7377+
);
73867378
return {
73877379
...snapshot,
73887380
projection: { ...snapshot.projection, visibleTurnItems },

‎apps/server/src/orchestration-v2/threadHistoryPaging.test.ts‎

Lines changed: 93 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,53 @@ function makeProjection(visibleTurnItems: OrchestrationV2ProjectedTurnItem[]) {
176176
}
177177

178178
describe("threadHistoryPaging", () => {
179+
it("bounds a single human turn and pages every item exactly once", () => {
180+
const items = Array.from({ length: 180 }, (_, index) => makeRow(index));
181+
items[0] = {
182+
...items[0]!,
183+
item: {
184+
...items[0]!.item,
185+
type: "user_message",
186+
createdBy: "user",
187+
creationSource: "web",
188+
inputIntent: "turn_start",
189+
messageId: MessageId.make("long-human-turn"),
190+
text: "Continue",
191+
attachments: [],
192+
},
193+
} as OrchestrationV2ProjectedTurnItem;
194+
const recent = selectRecentTimelineWindow({ items, snapshotSequence: 1 });
195+
expect(recent.items).toHaveLength(THREAD_HISTORY_PAGE_POLICY.maxItems);
196+
const loaded = [...recent.items];
197+
let cursor = recent.nextCursor;
198+
for (let pages = 0; cursor !== null; pages += 1) {
199+
expect(pages).toBeLessThan(items.length);
200+
const page = selectHistoryPageFromCursor({ items, cursor, snapshotSequence: 1 });
201+
expect(page.items.length).toBeGreaterThan(0);
202+
expect(page.items.length).toBeLessThanOrEqual(THREAD_HISTORY_PAGE_POLICY.maxItems);
203+
loaded.unshift(...page.items);
204+
cursor = page.nextCursor;
205+
}
206+
expect(loaded.map((row) => row.sourceItemId)).toEqual(items.map((row) => row.sourceItemId));
207+
208+
const large = items
209+
.slice(0, 12)
210+
.map((row) =>
211+
row.item.type === "command_execution"
212+
? { ...row, item: { ...row.item, output: "x".repeat(200_000) } }
213+
: row,
214+
);
215+
const bounded = buildBoundedThreadProjection({
216+
projection: makeProjection(large),
217+
snapshotSequence: 1,
218+
});
219+
expect(bounded.projection.visibleTurnItems.length).toBeLessThan(large.length);
220+
expect(bounded.payloadBudgetExceeded).toBe(false);
221+
expect(boundedTimelineEncodedBytes(bounded.projection)).toBeLessThanOrEqual(
222+
THREAD_HISTORY_PAGE_POLICY.maxEncodedBytes,
223+
);
224+
});
225+
179226
it("keeps background turns with user turns, with main's 150-turn fan-out ceiling", () => {
180227
const items = Array.from({ length: 161 }, (_, turn) => {
181228
const row = makeRow(turn * 2);
@@ -195,13 +242,15 @@ describe("threadHistoryPaging", () => {
195242
};
196243
return [prompt, makeRow(turn * 2 + 1)];
197244
}).flat();
198-
const first = selectRecentTimelineWindow({ items, snapshotSequence: 1 });
245+
const policy = { ...THREAD_HISTORY_PAGE_POLICY, maxItems: 1_000, maxEncodedBytes: 10_000_000 };
246+
const first = selectRecentTimelineWindow({ items, snapshotSequence: 1, policy });
199247
expect(first.items).toHaveLength(300);
200248
expect(first.items[0]?.sourceItemId).toBe("item-22");
201249
const older = selectHistoryPageFromCursor({
202250
items,
203251
cursor: first.nextCursor!,
204252
snapshotSequence: 1,
253+
policy,
205254
});
206255
expect(older.items).toHaveLength(22);
207256
expect(older.hasMoreHistory).toBe(false);
@@ -244,13 +293,15 @@ describe("threadHistoryPaging", () => {
244293
inRun(run * 4 + 2, run, notification(`Steer ${run}`)),
245294
inRun(run * 4 + 3, run),
246295
]).flat();
247-
const first = selectRecentTimelineWindow({ items, snapshotSequence: 1 });
296+
const policy = { ...THREAD_HISTORY_PAGE_POLICY, maxItems: 1_000, maxEncodedBytes: 10_000_000 };
297+
const first = selectRecentTimelineWindow({ items, snapshotSequence: 1, policy });
248298
expect(first.items).toHaveLength(150 * 4 - 1);
249299
expect(first.items[0]?.sourceItemId).toBe(`item-${11 * 4 + 1}`);
250300
const older = selectHistoryPageFromCursor({
251301
items,
252302
cursor: first.nextCursor!,
253303
snapshotSequence: 1,
304+
policy,
254305
});
255306
expect([...older.items, ...first.items].map((row) => row.sourceItemId)).toEqual(
256307
items.map((row) => row.sourceItemId),
@@ -288,26 +339,46 @@ describe("threadHistoryPaging", () => {
288339
});
289340

290341
it("always includes at least one pathological oversized item", () => {
291-
const items = [makeRow(0, { outputBytes: 2_000_000 }), makeRow(1, { outputBytes: 10 })];
292-
const page = selectRecentTimelineWindow({
293-
items,
294-
snapshotSequence: 1,
295-
policy: { maxItems: 50, maxEncodedBytes: 1_024 },
296-
});
297-
expect(page.items).toHaveLength(1);
298-
expect(page.items[0]?.sourceItemId).toBe("item-1");
299-
expect(page.hasMoreHistory).toBe(true);
300-
301-
const older = selectHistoryPageFromCursor({
302-
items,
303-
cursor: page.nextCursor!,
304-
snapshotSequence: 1,
305-
policy: { maxItems: 50, maxEncodedBytes: 1_024 },
306-
});
307-
expect(older.items).toHaveLength(1);
308-
expect(older.items[0]?.sourceItemId).toBe("item-0");
309-
expect(older.hasMoreHistory).toBe(false);
310-
expect(older.nextCursor).toBeNull();
342+
for (const humanTurn of [false, true]) {
343+
const oversized = makeRow(0, { outputBytes: 2_000_000 });
344+
const items = [
345+
humanTurn
346+
? ({
347+
...oversized,
348+
item: {
349+
...oversized.item,
350+
type: "user_message",
351+
createdBy: "user",
352+
creationSource: "web",
353+
inputIntent: "turn_start",
354+
messageId: MessageId.make("oversized-human-turn"),
355+
text: "x".repeat(2_000_000),
356+
attachments: [],
357+
},
358+
} as OrchestrationV2ProjectedTurnItem)
359+
: oversized,
360+
makeRow(1, { outputBytes: 10 }),
361+
];
362+
const policy = { ...THREAD_HISTORY_PAGE_POLICY, maxItems: 50, maxEncodedBytes: 1_024 };
363+
const page = selectRecentTimelineWindow({ items, snapshotSequence: 1, policy });
364+
expect(page.items).toHaveLength(1);
365+
expect(page.items[0]?.sourceItemId).toBe("item-1");
366+
expect(page.hasMoreHistory).toBe(true);
367+
368+
const older = selectHistoryPageFromCursor({
369+
items,
370+
cursor: page.nextCursor!,
371+
snapshotSequence: 1,
372+
policy,
373+
});
374+
expect(older.items).toHaveLength(1);
375+
expect(older.items[0]?.sourceItemId).toBe("item-0");
376+
expect(older.hasMoreHistory).toBe(false);
377+
expect(older.nextCursor).toBeNull();
378+
expect([...older.items, ...page.items].map((row) => row.sourceItemId)).toEqual(
379+
items.map((row) => row.sourceItemId),
380+
);
381+
}
311382
});
312383

313384
it("recovers identity-miss cursors when position is past a shrunken timeline", () => {

‎apps/server/src/orchestration-v2/threadHistoryPaging.ts‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,8 @@ import type {
77
} from "@t3tools/contracts";
88

99
/**
10-
* Item/byte budgets only apply to histories without turn starts; tool activity
11-
* must not split an ordinary conversation turn.
10+
* Bound every history window by rows and bytes, including a single long human
11+
* turn. Turn limits can stop a window earlier.
1212
*/
1313
export const THREAD_HISTORY_PAGE_POLICY = {
1414
maxUserTurns: 10,
@@ -210,8 +210,7 @@ export function isThreadHistoryUserTurn(item: HistoryRow["item"]): boolean {
210210
}
211211

212212
/**
213-
* Histories without turn starts use row budgets and always admit at least one
214-
* row so oversized items cannot deadlock paging.
213+
* Always admit at least one row so oversized items cannot deadlock paging.
215214
*/
216215
function selectOlderTimelinePage<Row extends HistoryRow>(input: {
217216
readonly items: ReadonlyArray<Row>;
@@ -238,12 +237,13 @@ function selectOlderTimelinePage<Row extends HistoryRow>(input: {
238237
: undefined;
239238
for (let index = end - 1; index >= 0; index -= 1) {
240239
const row = input.items[index]!;
241-
const rowBytes = turnLimit === undefined ? rowCost(row) : 0;
240+
const rowBytes = rowCost(row);
242241
if (
243242
selected.length > 0 &&
244-
(turnLimit === undefined
245-
? selected.length >= policy.maxItems || encodedBytes + rowBytes > policy.maxEncodedBytes
246-
: userTurns >= turnLimit || rawTurns >= THREAD_HISTORY_MAX_RAW_TURNS)
243+
(selected.length >= policy.maxItems ||
244+
encodedBytes + rowBytes > policy.maxEncodedBytes ||
245+
(turnLimit !== undefined &&
246+
(userTurns >= turnLimit || rawTurns >= THREAD_HISTORY_MAX_RAW_TURNS)))
247247
) {
248248
break;
249249
}

‎packages/client-runtime/src/platform/orchestrationCache.test.ts‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -74,14 +74,20 @@ describe("orchestration cache envelopes", () => {
7474
expect(decoded.visibleTurnItems.find((entry) => entry.item.id === item.id)?.item).toEqual(item);
7575
});
7676

77-
it("rejects thread snapshots saved before queued-run items were kept, but not shells", () => {
78-
const staleThread = encodeStoredThreadSnapshotJson({
77+
it("rejects thread snapshots saved before human history was bounded, but not shells", () => {
78+
const currentThread = encodeStoredThreadSnapshotJson({
7979
schemaVersion: ORCHESTRATION_THREAD_CACHE_SCHEMA_VERSION,
8080
environmentId,
8181
threadId: v2ThreadId,
8282
snapshot: { snapshotSequence: 4, projection: v2Projection },
83-
}).replace(`"schemaVersion":${ORCHESTRATION_THREAD_CACHE_SCHEMA_VERSION}`, `"schemaVersion":3`);
84-
expect(() => decodeStoredThreadSnapshotJson(staleThread)).toThrow();
83+
});
84+
for (const staleVersion of [3, 4]) {
85+
const staleThread = currentThread.replace(
86+
`"schemaVersion":${ORCHESTRATION_THREAD_CACHE_SCHEMA_VERSION}`,
87+
`"schemaVersion":${staleVersion}`,
88+
);
89+
expect(() => decodeStoredThreadSnapshotJson(staleThread)).toThrow();
90+
}
8591
const shell = encodeStoredShellSnapshot({
8692
schemaVersion: ORCHESTRATION_CACHE_SCHEMA_VERSION,
8793
environmentId,

‎packages/client-runtime/src/platform/orchestrationCache.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,8 @@ import * as Option from "effect/Option";
1010
import * as Schema from "effect/Schema";
1111

1212
export const ORCHESTRATION_CACHE_SCHEMA_VERSION = 3 as const;
13-
export const ORCHESTRATION_THREAD_CACHE_SCHEMA_VERSION = 4 as const;
13+
// Older caches may contain human turns that bypassed server history budgets.
14+
export const ORCHESTRATION_THREAD_CACHE_SCHEMA_VERSION = 5 as const;
1415

1516
export const StoredOrchestrationShellSnapshot = Schema.Struct({
1617
schemaVersion: Schema.Literal(ORCHESTRATION_CACHE_SCHEMA_VERSION),

0 commit comments

Comments
 (0)