Skip to content

Commit 7d3635e

Browse files
t3dotggclaude
andauthored
perf(server): oversized transcript exports fail fast and thread opens skip unused handoff bodies (#34547)
Co-authored-by: Claude Fable 5.2 <[email protected]>
1 parent 8b2763f commit 7d3635e

11 files changed

Lines changed: 364 additions & 21 deletions

‎apps/server/integration/TransferBudgetReport.integration.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,7 @@ const FLEET_SHELL_BUDGET = {
106106
// About 14% above the observed 75 KB. The window holds the working run only: 2 nodes, 2 subagents,
107107
// 1 plan, 1 provider turn and the last finished turn's checkpoint. When finished runs ride along
108108
// it holds 82 nodes, 82 subagents, 41 plans, 41 provider turns, 40 checkpoints and 740 KB.
109+
// The working run's ready handoff ships without its 256 KB history; with it the open is 339 KB.
109110
const STALE_TODO_BUDGET = {
110111
decodedBytes: 85_000,
111112
rows: 75,

‎apps/server/integration/TransferBudgetV2Fixture.integration.ts‎

Lines changed: 46 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import {
22
CheckpointId,
33
CheckpointRef,
44
CheckpointScopeId,
5+
ContextHandoffId,
56
EventId,
67
MessageId,
78
NodeId,
@@ -446,6 +447,7 @@ export const STALE_TODO_THREAD_ID = ThreadId.make("transfer-budget-stale-todo-li
446447
export const STALE_TODO_SETTLED_RUNS = 40;
447448
export const STALE_TODO_SUBAGENTS_PER_RUN = 2;
448449
export const STALE_TODO_LIVE_ROWS = 600;
450+
const STALE_TODO_HANDOFF_HISTORY_BYTES = 256 * 1024;
449451
const staleTodoCheckpointId = (index: number) =>
450452
CheckpointId.make(`checkpoint:thread:${STALE_TODO_THREAD_ID}:ordinal:${index + 1}`);
451453
/** The checkpoint clients read the last finished turn's diff from. */
@@ -742,7 +744,50 @@ export function staleTodoListThreadBatches(
742744
exitCode: 0,
743745
},
744746
});
745-
if (live) return events;
747+
if (live) {
748+
// A ready handoff rides along with every window. Its history and delivery are for the
749+
// provider turn start and must not reach a client.
750+
const earlierItemId = TurnItemId.make(`turn-item:stale-todo:${index}:handoff-source`);
751+
events.push({
752+
...common,
753+
id: EventId.make(`stale-todo-handoff-${index}`),
754+
type: "context-handoff.updated",
755+
payload: {
756+
id: ContextHandoffId.make(`handoff:run:${runKey}`),
757+
transferId: null,
758+
threadId,
759+
targetRunId: runId,
760+
fromProviderThreadIds: [providerThreadId],
761+
toProviderThreadId: providerThreadId,
762+
coveredRunOrdinals: { from: 1, to: index },
763+
strategy: "full_thread_summary",
764+
status: "ready",
765+
summaryMessageId: null,
766+
summaryText: text(index, 6, 1_024),
767+
history: {
768+
messages: [
769+
{
770+
role: "user",
771+
text: text(index, 7, STALE_TODO_HANDOFF_HISTORY_BYTES),
772+
threadId,
773+
runId: null,
774+
itemId: earlierItemId,
775+
providerThreadId,
776+
status: "completed",
777+
kind: "user_message",
778+
},
779+
],
780+
coverage: "full",
781+
omittedItems: 0,
782+
},
783+
delivery: { nativeThreadId: "native", status: "injected", itemIds: [earlierItemId] },
784+
createdByProviderInstanceId: providerInstanceId,
785+
createdAt: now,
786+
updatedAt: now,
787+
},
788+
});
789+
return events;
790+
}
746791

747792
message("assistant", text(index, 5, 1_200));
748793
const checkpointId = staleTodoCheckpointId(index);

‎apps/server/integration/transferBudgetV2.integration.test.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -448,6 +448,8 @@ it.live("reports V2 thread HTTP and WebSocket transfer budgets", () =>
448448
staleTodo.subagents.filter((subagent) => subagent.runId === STALE_TODO_LIVE_RUN_ID),
449449
STALE_TODO_SUBAGENTS_PER_RUN,
450450
);
451+
// The byte cap below also passes when the window loses the handoff.
452+
assert.lengthOf(staleTodo.contextHandoffs, 1);
451453
const staleTodoColdOpen = {
452454
http: staleTodoHttp,
453455
rows: staleTodo.visibleTurnItems.length,

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -428,6 +428,10 @@ export interface OrchestratorV2Shape {
428428
options: ProjectionTimelinePageOptions,
429429
) => Effect.Effect<ProjectionTimelinePage, OrchestratorProjectionError>;
430430
readonly getMessageCount: (threadId: ThreadId) => Effect.Effect<number, OrchestratorV2Error>;
431+
readonly getTranscriptByteFloor: (
432+
threadId: ThreadId,
433+
limit: number,
434+
) => Effect.Effect<number, OrchestratorV2Error>;
431435
readonly getTurnItem: (input: {
432436
readonly threadId: ThreadId;
433437
readonly itemId: TurnItemId;
@@ -15418,6 +15422,13 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1541815422
.getMessageCount(threadId)
1541915423
.pipe(Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause })));
1542015424
}),
15425+
getTranscriptByteFloor: Effect.fn("OrchestratorV2.getTranscriptByteFloor")(
15426+
function (threadId, limit) {
15427+
return projectionStore
15428+
.getTranscriptByteFloor(threadId, limit)
15429+
.pipe(Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause })));
15430+
},
15431+
),
1542115432
getTurnItem: Effect.fn("OrchestratorV2.getTurnItem")(function (input) {
1542215433
return projectionStore
1542315434
.getTurnItem(input)

‎apps/server/src/orchestration-v2/ProjectionStore.test.ts‎

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import {
99
ContextTransferId,
1010
CheckpointRef,
1111
CheckpointScopeId,
12+
ContextHandoffId,
1213
MessageId,
1314
type ModelSelection,
1415
type OrchestrationV2ProviderThread,
@@ -55,6 +56,7 @@ import {
5556
THREAD_HISTORY_PAGE_POLICY,
5657
THREAD_HISTORY_TURN_SNAPSHOT_ROW_LIMIT,
5758
} from "./threadHistoryPaging.ts";
59+
import { buildBoundedThreadStreamSnapshot } from "./ThreadStream.ts";
5860
import { limitRecoveryCommand } from "./UsageLimitRecoveryWorker.ts";
5961

6062
import { layerReaderDatabase, traceSqlRouting } from "../persistence/testkit/SqliteRouting.ts";
@@ -2594,6 +2596,142 @@ it.effect("SQL snapshot windows keep cards for requests still pending", () =>
25942596
}).pipe(Effect.provide(layerTest)),
25952597
);
25962598

2599+
it.effect("SQL snapshot windows leave handoff bodies in SQLite without changing the wire", () =>
2600+
Effect.gen(function* () {
2601+
const store = yield* ProjectionStore.ProjectionStoreV2;
2602+
const now = yield* DateTime.now;
2603+
const threadId = ThreadId.make("thread:handoff-window");
2604+
yield* store.apply({
2605+
id: EventId.make("event:handoff-window:thread"),
2606+
type: "thread.created",
2607+
threadId,
2608+
occurredAt: now,
2609+
payload: {
2610+
createdBy: "user",
2611+
creationSource: "web",
2612+
id: threadId,
2613+
projectId: ProjectId.make("project:handoff-window"),
2614+
title: "Handoff window",
2615+
providerInstanceId,
2616+
modelSelection,
2617+
runtimeMode: "full-access",
2618+
interactionMode: "default",
2619+
branch: null,
2620+
worktreePath: null,
2621+
activeProviderThreadId: null,
2622+
lineage: { parentThreadId: null, relationshipToParent: null, rootThreadId: threadId },
2623+
forkedFrom: null,
2624+
createdAt: now,
2625+
updatedAt: now,
2626+
archivedAt: null,
2627+
settledOverride: null,
2628+
settledAt: null,
2629+
lastVisitedAt: null,
2630+
deletedAt: null,
2631+
},
2632+
});
2633+
const providerThreadId = ProviderThreadId.make("provider-thread:handoff-window");
2634+
const earlierItemId = TurnItemId.make("item:handoff-window:earlier");
2635+
const handoff = (id: string, status: "ready" | "superseded") => ({
2636+
id: ContextHandoffId.make(`handoff:handoff-window:${id}`),
2637+
transferId: null,
2638+
threadId,
2639+
targetRunId: RunId.make("run:handoff-window"),
2640+
fromProviderThreadIds: [providerThreadId],
2641+
toProviderThreadId: providerThreadId,
2642+
coveredRunOrdinals: { from: 1, to: 1 },
2643+
strategy: "full_thread_summary" as const,
2644+
status,
2645+
summaryMessageId: null,
2646+
summaryText: `Summary ${id}`,
2647+
history: {
2648+
messages: [
2649+
{
2650+
role: "user" as const,
2651+
text: "Earlier request ".repeat(1_000),
2652+
threadId,
2653+
runId: null,
2654+
itemId: earlierItemId,
2655+
providerThreadId,
2656+
status: "completed",
2657+
kind: "user_message",
2658+
},
2659+
],
2660+
coverage: "full",
2661+
omittedItems: 0,
2662+
},
2663+
delivery: { nativeThreadId: "native", status: "injected" as const, itemIds: [earlierItemId] },
2664+
createdByProviderInstanceId: providerInstanceId,
2665+
createdAt: now,
2666+
updatedAt: now,
2667+
});
2668+
// The window keeps a ready handoff and one its rows show; the third is outside it.
2669+
const handoffs = [
2670+
handoff("ready", "ready"),
2671+
handoff("shown", "superseded"),
2672+
handoff("old", "superseded"),
2673+
];
2674+
for (const payload of handoffs) {
2675+
yield* store.apply({
2676+
id: EventId.make(`event:${payload.id}`),
2677+
type: "context-handoff.updated",
2678+
threadId,
2679+
occurredAt: now,
2680+
payload,
2681+
});
2682+
}
2683+
yield* store.apply({
2684+
id: EventId.make("event:handoff-window:item"),
2685+
type: "turn-item.updated",
2686+
threadId,
2687+
occurredAt: now,
2688+
payload: {
2689+
id: TurnItemId.make("item:handoff-window:handoff"),
2690+
threadId,
2691+
runId: null,
2692+
nodeId: null,
2693+
providerThreadId: null,
2694+
providerTurnId: null,
2695+
nativeItemRef: null,
2696+
parentItemId: null,
2697+
ordinal: 1,
2698+
status: "completed",
2699+
title: null,
2700+
startedAt: now,
2701+
completedAt: now,
2702+
updatedAt: now,
2703+
type: "handoff",
2704+
contextHandoffId: handoffs[1]!.id,
2705+
fromProviderThreadIds: [providerThreadId],
2706+
toProviderThreadId: providerThreadId,
2707+
fromProviderInstanceIds: [providerInstanceId],
2708+
toProviderInstanceId: providerInstanceId,
2709+
strategy: "full_thread_summary",
2710+
},
2711+
});
2712+
2713+
const window = yield* store.getThreadSnapshotWindow(threadId, { rowLimit: 10 });
2714+
const windowHandoffs = window.projection.contextHandoffs;
2715+
assert.deepEqual(
2716+
windowHandoffs.map((row) => row.id),
2717+
[handoffs[0]!.id, handoffs[1]!.id],
2718+
);
2719+
for (const row of windowHandoffs) {
2720+
assert.isUndefined(row.history);
2721+
assert.isUndefined(row.delivery);
2722+
}
2723+
const full = yield* store.getThreadSnapshot(threadId);
2724+
assert.deepEqual(full.projection.contextHandoffs, handoffs);
2725+
assert.deepEqual(
2726+
buildBoundedThreadStreamSnapshot(window),
2727+
buildBoundedThreadStreamSnapshot({
2728+
...window,
2729+
projection: { ...window.projection, contextHandoffs: handoffs.slice(0, 2) },
2730+
}),
2731+
);
2732+
}).pipe(Effect.provide(layerTest)),
2733+
);
2734+
25972735
it.effect("lists only active subagent IDs without loading shell data", () =>
25982736
Effect.gen(function* () {
25992737
const store = yield* ProjectionStore.ProjectionStoreV2;

‎apps/server/src/orchestration-v2/ProjectionStore.ts‎

Lines changed: 45 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -472,6 +472,11 @@ export interface ProjectionStoreV2Shape {
472472
options: ProjectionTimelinePageOptions,
473473
) => Effect.Effect<ProjectionTimelinePage, ProjectionStoreV2Error>;
474474
readonly getMessageCount: (threadId: ThreadId) => Effect.Effect<number, ProjectionStoreV2Error>;
475+
/** Bytes of tool output a transcript export is sure to hold, or 0 when it can fit in `limit`. */
476+
readonly getTranscriptByteFloor: (
477+
threadId: ThreadId,
478+
limit: number,
479+
) => Effect.Effect<number, ProjectionStoreV2Error>;
475480
readonly getSubagentCount: (threadId: ThreadId) => Effect.Effect<number, ProjectionStoreV2Error>;
476481
/** Child threads of the thread's app-owned tasks, in task order, without decoding the tasks. */
477482
readonly listAppOwnedChildThreadIds: (
@@ -716,6 +721,7 @@ export interface ProjectionStoreV2Shape {
716721
},
717722
ProjectionStoreV2Error
718723
>;
724+
/** Context handoffs come without `history` and `delivery`; no window reader uses them. */
719725
readonly getThreadSnapshotWindow: (
720726
threadId: ThreadId,
721727
options: {
@@ -4003,7 +4009,9 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
40034009
WHERE thread_id = ${threadId} ORDER BY rowid ASC
40044010
`
40054011
: sql<PayloadRow>`
4006-
SELECT payload_json FROM orchestration_v2_projection_context_handoffs AS handoff
4012+
-- Megabytes on a long thread, and projectContextHandoffForWire drops both.
4013+
SELECT json_remove(payload_json, '$.history', '$.delivery') AS payload_json
4014+
FROM orchestration_v2_projection_context_handoffs AS handoff
40074015
WHERE handoff.thread_id = ${threadId}
40084016
AND (
40094017
handoff.status IN ('pending', 'ready')
@@ -5915,6 +5923,41 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
59155923
Effect.map((rows) => rows[0]?.count ?? 0),
59165924
Effect.mapError(controlReadError(threadId)),
59175925
);
5926+
// Tool output reaches the export unchanged, so the bytes of its text, also inside a structured
5927+
// result, are a floor for the export. Other stored bytes are not: decoding drops fields. A
5928+
// rolled back run is the only way the export hides a tool row, and rows a fork inherits only
5929+
// add to it. The output sum parses payloads, so it runs only when the tool rows exceed `limit`.
5930+
const getTranscriptByteFloor: ProjectionStoreV2Shape["getTranscriptByteFloor"] = (
5931+
threadId,
5932+
limit,
5933+
) =>
5934+
sql<{ bytes: number }>`
5935+
WITH tool_rows AS (
5936+
SELECT payload_json FROM orchestration_v2_projection_turn_items
5937+
WHERE thread_id = ${threadId}
5938+
AND type IN ('command_execution', 'dynamic_tool')
5939+
AND (run_id IS NULL OR run_id NOT IN (
5940+
SELECT run_id FROM orchestration_v2_projection_runs
5941+
WHERE thread_id = ${threadId} AND status = 'rolled_back'
5942+
))
5943+
)
5944+
SELECT CASE
5945+
WHEN (SELECT COALESCE(SUM(octet_length(payload_json)), 0) FROM tool_rows) <= ${limit}
5946+
THEN 0
5947+
ELSE (
5948+
-- json_valid is false past SQLite's nesting limit, where json_tree would fail.
5949+
SELECT COALESCE(SUM(CASE WHEN json_valid(payload_json) THEN (
5950+
SELECT COALESCE(SUM(octet_length(leaf.value)), 0)
5951+
FROM json_tree(tool_rows.payload_json, '$.output') AS leaf
5952+
WHERE leaf.type = 'text'
5953+
) ELSE 0 END), 0)
5954+
FROM tool_rows
5955+
)
5956+
END AS bytes
5957+
`.pipe(
5958+
Effect.map((rows) => rows[0]?.bytes ?? 0),
5959+
Effect.mapError(controlReadError(threadId)),
5960+
);
59185961
const getSubagentCount: ProjectionStoreV2Shape["getSubagentCount"] = (threadId) =>
59195962
sql<{
59205963
count: number;
@@ -7995,6 +8038,7 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
79958038
hasUnpairedRunInterruptRequest: readOnReader(hasUnpairedRunInterruptRequest),
79968039
isNodeStored: readOnReader(isNodeStored),
79978040
getMessageCount: readOnReader(getMessageCount),
8041+
getTranscriptByteFloor: readOnReader(getTranscriptByteFloor),
79988042
getSubagentCount: readOnReader(getSubagentCount),
79998043
listAppOwnedChildThreadIds: readOnReader(listAppOwnedChildThreadIds),
80008044
hasPendingDelegatedCompletion: readOnReader(hasPendingDelegatedCompletion),

‎apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -239,6 +239,7 @@ it.effect(
239239
getThreadHistoryPage: () => Effect.die("unused"),
240240
getTimelinePage: () => Effect.die("Unused timeline read"),
241241
getMessageCount: () => Effect.die("unused message count"),
242+
getTranscriptByteFloor: () => Effect.die("unused transcript byte floor"),
242243
hasPendingDelegatedCompletion: () => Effect.die("unused hasPendingDelegatedCompletion"),
243244
getSubagentCount: () => Effect.die("unused subagent count"),
244245
listAppOwnedChildThreadIds: () => Effect.die("unused listAppOwnedChildThreadIds"),

‎apps/server/src/orchestration-v2/ThreadManagementService.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -890,6 +890,7 @@ it.effect(
890890
Layer.provide(Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({})),
891891
Layer.provide(
892892
Layer.mock(Orchestrator.OrchestratorV2)({
893+
getTranscriptByteFloor: () => Effect.succeed(0),
893894
getThreadSnapshot: (threadId) =>
894895
Queue.offer(started, threadId).pipe(
895896
Effect.andThen(Deferred.await(gates.get(threadId)!)),

‎apps/server/src/orchestration-v2/ThreadManagementService.ts‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -687,7 +687,15 @@ const make = Effect.gen(function* () {
687687
const transcriptExportPermits = yield* Semaphore.make(2);
688688
const getThreadTranscript = Effect.fn("orchestrationV2.threadManagement.getThreadTranscript")(
689689
function* (threadId: ThreadId) {
690-
const { projection } = yield* getThreadSnapshot(threadId);
690+
yield* ensureProjectionTranscript(threadId);
691+
const byteFloor = yield* orchestrator.getTranscriptByteFloor(
692+
threadId,
693+
PROVIDER_SEND_TURN_MAX_FILE_BYTES,
694+
);
695+
if (byteFloor > PROVIDER_SEND_TURN_MAX_FILE_BYTES) {
696+
return yield* new ThreadTranscriptTooLargeError({ threadId });
697+
}
698+
const { projection } = yield* orchestrator.getThreadSnapshot(threadId);
691699
const environment = yield* ServerEnvironment.ServerEnvironmentIdentity;
692700
const transcript = {
693701
threadId: projection.thread.id,

0 commit comments

Comments
 (0)