Skip to content

Commit c138163

Browse files
t3dotggclaude
andauthored
perf: cheaper shell refreshes, one copy of Codex streaming text, no MCP wait polling (pingdotgg#15033)
Co-authored-by: Claude Opus 5.5 (1M context) <[email protected]>
1 parent 2afe87e commit c138163

21 files changed

Lines changed: 525 additions & 128 deletions

‎apps/server/src/mcp/OrchestratorMcpService.ts‎

Lines changed: 40 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,7 @@ import * as Effect from "effect/Effect";
6363
import * as Layer from "effect/Layer";
6464
import * as Option from "effect/Option";
6565
import * as Schema from "effect/Schema";
66+
import * as Stream from "effect/Stream";
6667

6768
import * as ProviderAdapterRegistry from "../orchestration-v2/ProviderAdapterRegistry.ts";
6869
import {
@@ -81,7 +82,14 @@ import {
8182

8283
const DEFAULT_WAIT_TIMEOUT_MS = 10 * 60 * 1_000;
8384
const MAX_WAIT_TIMEOUT_MS = 60 * 60 * 1_000;
84-
const TASK_POLL_INTERVAL_MS = 50;
85+
// Events that can make a delegated task terminal: the parent's task record,
86+
// and the child's runs, nested tasks, and pending provider background work.
87+
const TASK_WAKE_EVENTS = [
88+
{ thread: "parent", eventType: "subagent.updated" },
89+
{ thread: "child", eventType: "run.updated" },
90+
{ thread: "child", eventType: "subagent.updated" },
91+
{ thread: "child", eventType: "provider-thread.updated" },
92+
] as const;
8593
const DEFAULT_THREAD_LIST_LIMIT = 50;
8694
const DEFAULT_THREAD_READ_LIMIT = 50;
8795
const DEFAULT_THREAD_RUN_LIMIT = 10;
@@ -1298,14 +1306,39 @@ const make = Effect.gen(function* () {
12981306
return response;
12991307
});
13001308

1309+
// Re-read the task only when an event on the parent or child thread can
1310+
// change its status, instead of polling the projections every 50 ms.
13011311
const waitForTask = (scope: McpThreadInvocationScope, taskId: NodeId, timeoutMs: number) =>
13021312
Effect.gen(function* () {
1303-
while (true) {
1304-
const result = yield* readTask(scope, taskId, false, true);
1305-
if (isTerminalTaskStatus(result.status)) return result;
1306-
yield* Effect.sleep(Duration.millis(TASK_POLL_INTERVAL_MS));
1307-
}
1308-
}).pipe(Effect.timeoutOption(Duration.millis(timeoutMs)));
1313+
const streamError = (error: unknown) =>
1314+
failure(
1315+
"orchestration_error",
1316+
`Unable to watch delegated task ${taskId}: ${errorMessage(error)}`,
1317+
);
1318+
// Sequences are global, so one cursor taken before the first read
1319+
// replays anything either thread records after it.
1320+
const afterSequence = yield* threadManagement
1321+
.getThreadEventSequence(scope.thread.threadId)
1322+
.pipe(Effect.mapError(streamError));
1323+
const initial = yield* readTask(scope, taskId, false, true);
1324+
if (isTerminalTaskStatus(initial.status)) return Option.some(initial);
1325+
// One stream per event type, so transcript events never fill a buffer.
1326+
return yield* Stream.mergeAll(
1327+
TASK_WAKE_EVENTS.map(({ thread, eventType }) =>
1328+
threadManagement.streamStoredEventsFrom({
1329+
threadId: thread === "parent" ? scope.thread.threadId : initial.childThreadId,
1330+
afterSequence,
1331+
eventType,
1332+
}),
1333+
),
1334+
{ concurrency: "unbounded" },
1335+
).pipe(
1336+
Stream.mapError(streamError),
1337+
Stream.mapEffect(() => readTask(scope, taskId, false, true)),
1338+
Stream.filter((result) => isTerminalTaskStatus(result.status)),
1339+
Stream.runHead,
1340+
);
1341+
}).pipe(Effect.timeoutOption(Duration.millis(timeoutMs)), Effect.map(Option.flatten));
13091342

13101343
/**
13111344
* A scheduled task the caller may change: one whose modes are no broader

‎apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts‎

Lines changed: 50 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2291,6 +2291,12 @@ describe("CodexAdapterV2 post-settle continuation", () => {
22912291
(event): event is Extract<ProviderAdapterV2Event, { type: "message.updated" }> =>
22922292
event.type === "message.updated" && event.message.role === "assistant",
22932293
);
2294+
const assistantTurnItems = (events: ReadonlyArray<ProviderAdapterV2Event>) =>
2295+
events.flatMap((event) =>
2296+
event.type === "turn_item.updated" && event.turnItem.type === "assistant_message"
2297+
? [event.turnItem]
2298+
: [],
2299+
);
22942300

22952301
it.effect("keeps an asynchronous Codex question actionable after the turn completes", () =>
22962302
Effect.scoped(
@@ -3427,6 +3433,49 @@ describe("CodexAdapterV2 post-settle continuation", () => {
34273433
),
34283434
);
34293435

3436+
it.effect("streams text on the turn item and sends the message once, when it completes", () =>
3437+
Effect.scoped(
3438+
Effect.gen(function* () {
3439+
const transcript = finalAnswerTranscript("codex-streamed-message-once", [
3440+
{ id: "answer", text: "CODEX_RECOVERY_OK", streamed: true, completionDelayMs: 100 },
3441+
]);
3442+
const harness = yield* makeCodexReplayHarness(transcript);
3443+
const now = yield* DateTime.now;
3444+
3445+
yield* harness.runtime.startTurn(
3446+
makeCodexTestTurnInput({
3447+
threadId: harness.threadId,
3448+
providerThread: harness.providerThread,
3449+
now,
3450+
attemptId: RunAttemptId.make("attempt-codex-streamed-message-once"),
3451+
text: "Reply with the requested recovery marker.",
3452+
}),
3453+
);
3454+
yield* Effect.yieldNow;
3455+
yield* TestClock.adjust("50 millis");
3456+
yield* awaitUntil(
3457+
() => assistantTurnItems(harness.events).length === 1,
3458+
"streamed turn item",
3459+
);
3460+
assert.deepEqual(
3461+
assistantTurnItems(harness.events).map(({ text, streaming }) => ({ text, streaming })),
3462+
[{ text: "CODEX_RECOVERY_OK", streaming: true }],
3463+
);
3464+
assert.deepEqual(assistantMessages(harness.events), []);
3465+
3466+
yield* TestClock.adjust("50 millis");
3467+
yield* awaitUntil(() => harness.terminalEvents().length === 1, "root turn terminal");
3468+
assert.deepEqual(
3469+
assistantMessages(harness.events).map(({ message }) => ({
3470+
text: message.text,
3471+
streaming: message.streaming,
3472+
})),
3473+
[{ text: "CODEX_RECOVERY_OK", streaming: false }],
3474+
);
3475+
}).pipe(Effect.provide(Layer.merge(IdAllocator.layer, NodeServices.layer))),
3476+
),
3477+
);
3478+
34303479
it.effect("suppresses a later streamed duplicate final answer", () =>
34313480
Effect.scoped(
34323481
Effect.gen(function* () {
@@ -3578,10 +3627,7 @@ describe("CodexAdapterV2 post-settle continuation", () => {
35783627
yield* TestClock.adjust("50 millis");
35793628
yield* Effect.yieldNow;
35803629

3581-
assert.equal(
3582-
new Set(assistantMessages(harness.events).map((event) => event.message.id)).size,
3583-
1,
3584-
);
3630+
assert.equal(new Set(assistantTurnItems(harness.events).map((item) => item.id)).size, 1);
35853631

35863632
yield* TestClock.adjust("50 millis");
35873633
yield* awaitUntil(() => harness.terminalEvents().length === 1, "root turn terminal");

‎apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3180,11 +3180,16 @@ export function makeCodexAdapterV2(adapterOptions: CodexAdapterV2Options): Provi
31803180
driver: CODEX_PROVIDER,
31813181
node: artifacts.node,
31823182
});
3183-
yield* emitProviderEvent({
3184-
type: "message.updated",
3185-
driver: CODEX_PROVIDER,
3186-
message: artifacts.message,
3187-
});
3183+
// Clients render streaming text from the turn item, so the
3184+
// message only carries the final text. flushTurn completes every
3185+
// buffered item when a turn ends, interrupted or not.
3186+
if (update.completed) {
3187+
yield* emitProviderEvent({
3188+
type: "message.updated",
3189+
driver: CODEX_PROVIDER,
3190+
message: artifacts.message,
3191+
});
3192+
}
31883193
yield* emitProviderEvent({
31893194
type: "turn_item.updated",
31903195
driver: CODEX_PROVIDER,

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -316,6 +316,8 @@ export interface OrchestratorV2Shape {
316316
readonly streamStoredEventsFrom: (input?: {
317317
readonly threadId?: ThreadId;
318318
readonly afterSequence?: number;
319+
/** Keep only this type, before the bounded buffer retains anything. */
320+
readonly eventType?: OrchestrationV2DomainEvent["type"];
319321
}) => Stream.Stream<OrchestrationV2StoredEvent, OrchestratorV2Error>;
320322
readonly streamDomainEvents: Stream.Stream<OrchestrationV2DomainEvent, OrchestratorV2Error>;
321323
}

‎apps/server/src/orchestration-v2/ProjectionStore.test.ts‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3762,6 +3762,21 @@ it.layer(TestLayer)("ProjectionStoreV2", (it) => {
37623762
(yield* projectionStore.getThreadProjection(sourceThreadId)).messages,
37633763
);
37643764
const targetAfterRollback = yield* projectionStore.getThreadProjection(targetThreadId);
3765+
// Both shell reads drop the rolled-back run from the source count, while
3766+
// the fork keeps the prefix it inherited.
3767+
const shellSnapshot = yield* projectionStore.getShellSnapshot();
3768+
for (const shell of [
3769+
yield* projectionStore.getThreadShell(sourceThreadId),
3770+
shellSnapshot.threads.find((thread) => thread.id === sourceThreadId),
3771+
]) {
3772+
assert.equal(shell?.itemCount, 2);
3773+
}
3774+
for (const shell of [
3775+
yield* projectionStore.getThreadShell(targetThreadId),
3776+
shellSnapshot.threads.find((thread) => thread.id === targetThreadId),
3777+
]) {
3778+
assert.equal(shell?.visibleItemCount, targetAfterRollback.visibleTurnItems.length);
3779+
}
37653780
const forwardPage = yield* projectionStore.getTimelinePage(targetThreadId, {
37663781
view: "activity",
37673782
limit: 2,

‎apps/server/src/orchestration-v2/ProjectionStore.ts‎

Lines changed: 42 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1550,6 +1550,25 @@ function itemCountThroughRun(input: {
15501550
return count;
15511551
}
15521552

1553+
/**
1554+
* Threads that some shell row forks from at a run. Only these need run
1555+
* ordinals and per-run item counts, to count the inherited prefix, so a thread
1556+
* nobody forked from never pays for a scan of its run history.
1557+
*/
1558+
function shellForkSourceIds(
1559+
rows: ReadonlyArray<{ readonly forked_from_run_source_thread_id: string | null }>,
1560+
): ReadonlyArray<ThreadId> {
1561+
return [
1562+
...new Set(
1563+
rows.flatMap((row) =>
1564+
row.forked_from_run_source_thread_id === null
1565+
? []
1566+
: [ThreadId.make(row.forked_from_run_source_thread_id)],
1567+
),
1568+
),
1569+
];
1570+
}
1571+
15531572
function visibleItemCountForShell(input: {
15541573
readonly threadId: ThreadId;
15551574
readonly statesByThreadId: ReadonlyMap<ThreadId, ShellThreadState>;
@@ -4938,13 +4957,19 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
49384957
AND plan.kind = 'proposed_plan'
49394958
AND plan.status = 'active'
49404959
) AS has_actionable_proposed_plan,
4960+
-- Count per run on the covering (thread_id, run_id) index, then
4961+
-- look up each run once, instead of one run lookup per item.
49414962
(
4942-
SELECT COUNT(*)
4943-
FROM orchestration_v2_projection_turn_items i
4963+
SELECT COALESCE(SUM(per_run.item_count), 0)
4964+
FROM (
4965+
SELECT i.run_id, COUNT(*) AS item_count
4966+
FROM orchestration_v2_projection_turn_items i
4967+
WHERE i.thread_id = t.thread_id
4968+
GROUP BY i.run_id
4969+
) per_run
49444970
LEFT JOIN orchestration_v2_projection_runs r
4945-
ON r.run_id = i.run_id
4946-
WHERE i.thread_id = t.thread_id
4947-
AND (i.run_id IS NULL OR r.status <> 'rolled_back')
4971+
ON r.run_id = per_run.run_id
4972+
WHERE per_run.run_id IS NULL OR r.status <> 'rolled_back'
49484973
) AS item_count,
49494974
(
49504975
SELECT COUNT(*)
@@ -5426,14 +5451,15 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
54265451
}
54275452
const threadRows = [...rowsByThreadId.values()];
54285453
const threadIds = [...rowsByThreadId.keys()];
5454+
const forkSourceIds = shellForkSourceIds(threadRows);
54295455
const readForThreadIds = <A>(
54305456
read: (ids: ReadonlyArray<ThreadId>) => Effect.Effect<ReadonlyArray<A>, unknown>,
5431-
) =>
5432-
threadIds.length === 0 ? Effect.succeed([] as ReadonlyArray<A>) : read(threadIds);
5457+
ids: ReadonlyArray<ThreadId> = threadIds,
5458+
) => (ids.length === 0 ? Effect.succeed([] as ReadonlyArray<A>) : read(ids));
54335459
const [runRows, itemCountRows, sequenceRows, providerThreadRows, pendingTurnItemRows] =
54345460
yield* Effect.all([
5435-
readForThreadIds(selectShellRunRows),
5436-
readForThreadIds(selectShellRunItemCounts),
5461+
readForThreadIds(selectShellRunRows, forkSourceIds),
5462+
readForThreadIds(selectShellRunItemCounts, forkSourceIds),
54375463
sql<{ readonly snapshot_sequence: number | null }>`
54385464
SELECT MAX(sequence) AS snapshot_sequence
54395465
FROM orchestration_events
@@ -5520,10 +5546,15 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
55205546
}
55215547

55225548
const threadIds = [...rowsByThreadId.keys()];
5549+
const forkSourceIds = shellForkSourceIds([...rowsByThreadId.values()]);
55235550
const [runRows, itemCountRows, providerThreadRows, pendingTurnItemRows] =
55245551
yield* Effect.all([
5525-
selectShellRunRows(threadIds),
5526-
selectShellRunItemCounts(threadIds),
5552+
forkSourceIds.length === 0
5553+
? Effect.succeed([] as ReadonlyArray<ShellRunRow>)
5554+
: selectShellRunRows(forkSourceIds),
5555+
forkSourceIds.length === 0
5556+
? Effect.succeed([] as ReadonlyArray<ShellRunItemCountRow>)
5557+
: selectShellRunItemCounts(forkSourceIds),
55275558
selectShellProviderThreadRows(threadIds),
55285559
selectShellPendingTurnItemRows(threadIds),
55295560
]);

0 commit comments

Comments
 (0)