Skip to content

Commit b7b14bf

Browse files
fix(server): remote task followers read only their own task
RemoteDelegation.start resumes every open remote task at once, and each follower read its parent's whole subagent list to find one task, then held it across a wait of up to ten minutes. A parent with 1,441 tasks (35 MB of payload) and 122 open ones made every restart decode ~170k records and hold 8 GB, so the server ran out of heap seconds after starting and crash-looped. getThreadRecords now takes a subagentIds filter. Followers, and task_status waits that re-read on every parent subagent.updated, read just their task. Against a copy of the affected database the startup fan-out drops from 8.2 GB retained over 21.6 s to 8 MB over 61 ms. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 570ffd5 commit b7b14bf

24 files changed

Lines changed: 461 additions & 142 deletions

‎apps/mobile/src/features/threads/ThreadAgentsSheet.tsx‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,8 +60,8 @@ export function ThreadAgentsSheet({ route }: StaticScreenProps<AgentsTarget>) {
6060
if (
6161
openInspector(target, {
6262
type: "thread",
63-
environmentId: target.environmentId,
64-
threadId: childThreadId,
63+
environmentId: child.environmentId,
64+
threadId: child.threadId,
6565
})
6666
)
6767
return;

‎apps/server/src/mcp/OrchestratorMcpService.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -369,6 +369,7 @@ describe("OrchestratorMcpService", () => {
369369
}),
370370
Layer.mock(ProjectService.ProjectService)({}),
371371
Layer.mock(SecretRequests.SecretRequests)({}),
372+
Layer.mock(RemoteDelegation.RemoteDelegation)({}),
372373
Layer.mock(ScheduledTaskService.ScheduledTaskService)({}),
373374
);
374375
const scope: McpInvocationScope = {

‎apps/server/src/mcp/OrchestratorMcpService.ts‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,6 @@ import {
5252
type OrchestratorMcpThreadWaitInput,
5353
type OrchestratorMcpThreadWaitResult,
5454
type ProviderInteractionMode,
55-
type ProviderInstanceId,
5655
type ProviderOptionDescriptor,
5756
type ProviderOptionSelection,
5857
ProviderRequestKind,
@@ -1024,6 +1023,22 @@ const make = Effect.gen(function* () {
10241023
),
10251024
);
10261025

1026+
// Task waits read on every wake of a parent that may hold thousands of
1027+
// tasks, so they read only the task they watch and what its status needs.
1028+
const loadTaskParent = (threadId: ThreadId, taskId: NodeId) =>
1029+
threadManagement
1030+
.getThreadRecords(threadId, ["runs", "subagents", "contextTransfers"], {
1031+
subagentIds: [taskId],
1032+
})
1033+
.pipe(
1034+
Effect.mapError((error) =>
1035+
failure(
1036+
"orchestration_error",
1037+
`Unable to read thread ${threadId}: ${errorMessage(error)}`,
1038+
),
1039+
),
1040+
);
1041+
10271042
const loadProjectThread = (
10281043
projectId: OrchestrationV2ThreadProjection["thread"]["projectId"],
10291044
threadId: ThreadId,
@@ -1376,7 +1391,7 @@ const make = Effect.gen(function* () {
13761391
acknowledgementOperation = "task-status-acknowledge",
13771392
): Effect.fn.Return<OrchestratorMcpDelegateTaskResult, OrchestratorMcpFailure> {
13781393
yield* requireCapability(scope);
1379-
const parentProjection = yield* loadProjection(scope.thread.threadId);
1394+
const parentProjection = yield* loadTaskParent(scope.thread.threadId, taskId);
13801395
const task = parentProjection.subagents.find(
13811396
(candidate) =>
13821397
candidate.id === taskId &&

‎apps/server/src/mcp/toolkits/orchestrator/tools.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ const OrchestratorCapabilitiesTool = Tool.make("orchestrator_capabilities", {
7373

7474
const DelegateTaskTool = Tool.make("delegate_task", {
7575
description:
76-
"Start one child agent with only `task`. No parent history. Separate checkout or PR per child: `isolation:'worktree'`. Keep `taskId` for `task_status` and `task_cancel`. `target.environmentId` runs it on a linked machine, in the project with this repository unless `target.projectId` names one.",
76+
"Start one child agent with only `task`. No parent history. Separate checkout or PR per child: `isolation:'worktree'`. Keep `taskId` for `task_status` and `task_cancel`.",
7777
parameters: OrchestratorMcpDelegateTaskInput,
7878
success: OrchestratorMcpDelegateTaskResult,
7979
failure: OrchestratorMcpFailure,

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 22 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ import {
4747
type OrchestrationV2ExecutionNode,
4848
type OrchestrationV2ProviderThread,
4949
type OrchestrationV2ProviderTurn,
50+
type OrchestrationV2RemoteTaskChild,
5051
type OrchestrationV2Run,
5152
type OrchestrationV2RunAttempt,
5253
type OrchestrationV2ThreadShell,
@@ -7759,17 +7760,19 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
77597760
command.stoppedThere === undefined
77607761
? task.remoteChild
77617762
: { ...task.remoteChild!, stoppedThere: true };
7762-
// The follower may report a result twice; the first one stands, and the
7763-
// repeat records nothing, which the dispatcher refuses. A stop there that
7764-
// reached the thread after the result is still recorded, but a run that
7765-
// ended by itself does not settle a stop the task here still owes.
7763+
// The first result stands; a repeated follower report records nothing, which
7764+
// the dispatcher refuses. A stop there after the result is still recorded, but
7765+
// a run that ended by itself does not settle a stop the task here still owes.
77667766
if (task.result !== null) {
7767+
// A Stop or cancel here that lost the race to the result still owes its stop.
7768+
if (command.stoppedThere === undefined) {
7769+
return yield* emitRemoteStopOwed(events, command, task, task.remoteChild!);
7770+
}
77677771
const owesStop =
77687772
task.status === "cancelled" ||
77697773
task.status === "interrupted" ||
77707774
task.remoteChild?.stoppedThere === false;
77717775
if (
7772-
command.stoppedThere === undefined ||
77737776
task.remoteChild?.stoppedThere === true ||
77747777
(owesStop && command.stoppedThere !== "stopped")
77757778
)
@@ -7911,6 +7914,18 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
79117914
cause: `Delegated task ${command.taskId} is not a finished remote task of thread ${command.parentThreadId}.`,
79127915
});
79137916
}
7917+
yield* emitRemoteStopOwed(events, command, task, task.remoteChild);
7918+
});
7919+
7920+
const emitRemoteStopOwed = Effect.fn("orchestrationV2.dispatch.remoteStopOwed")(function* (
7921+
events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
7922+
command: Extract<
7923+
OrchestrationV2InternalCommand,
7924+
{ readonly type: "delegated_task.remote.complete" | "delegated_task.remote.stop" }
7925+
>,
7926+
task: OrchestrationV2Subagent,
7927+
remoteChild: OrchestrationV2RemoteTaskChild,
7928+
) {
79147929
const now = yield* DateTime.now;
79157930
yield* emit(
79167931
events,
@@ -7925,9 +7940,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
79257940
payload: {
79267941
...task,
79277942
remoteChild: {
7928-
...task.remoteChild,
7943+
...remoteChild,
79297944
stoppedThere: false,
7930-
stopCount: (task.remoteChild.stopCount ?? 0) + 1,
7945+
stopCount: (remoteChild.stopCount ?? 0) + 1,
79317946
},
79327947
updatedAt: now,
79337948
},

‎apps/server/src/orchestration-v2/ProjectionStore.test.ts‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -964,6 +964,56 @@ const runlessNativeSubagentNeedsRecovery = Effect.gen(function* () {
964964
}
965965
});
966966

967+
const subagentRecordsSelectById = Effect.gen(function* () {
968+
const store = yield* ProjectionStore.ProjectionStoreV2;
969+
const threadId = yield* addRolledBackRecoveryCandidate("subagent-records-by-id");
970+
const now = yield* DateTime.now;
971+
const ids = [NodeId.make("node:subagent-by-id:a"), NodeId.make("node:subagent-by-id:b")];
972+
for (const id of ids) {
973+
yield* store.apply({
974+
id: EventId.make(`event:${id}`),
975+
type: "subagent.updated",
976+
threadId,
977+
nodeId: id,
978+
driver,
979+
providerInstanceId,
980+
occurredAt: now,
981+
payload: {
982+
id,
983+
threadId,
984+
runId: null,
985+
parentNodeId: NodeId.make("node:subagent-by-id:parent"),
986+
origin: "app_owned",
987+
createdBy: "agent",
988+
driver,
989+
providerInstanceId,
990+
providerThreadId: null,
991+
childThreadId: null,
992+
nativeTaskRef: null,
993+
prompt: "Do the work",
994+
title: "Delegated task",
995+
model: null,
996+
status: "running",
997+
result: null,
998+
startedAt: now,
999+
completedAt: null,
1000+
updatedAt: now,
1001+
},
1002+
});
1003+
}
1004+
const read = (subagentIds?: ReadonlyArray<NodeId>) =>
1005+
store
1006+
.getThreadRecords(threadId, ["subagents"], subagentIds === undefined ? {} : { subagentIds })
1007+
.pipe(Effect.map(({ subagents }) => subagents.map((task) => task.id)));
1008+
assert.deepEqual(yield* read(), ids);
1009+
assert.deepEqual(yield* read([ids[1]!]), [ids[1]]);
1010+
assert.deepEqual(yield* read([]), []);
1011+
});
1012+
1013+
it.effect("memory record reads select subagents by id", () =>
1014+
subagentRecordsSelectById.pipe(Effect.provide(ProjectionStore.layerMemory)),
1015+
);
1016+
9671017
const assertProjectScopedShells = Effect.fn("assertProjectScopedShells")(function* (
9681018
suffix: string,
9691019
) {
@@ -1392,6 +1442,7 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => {
13921442
assert.deepEqual(recovery?.restartCancelledBackgroundWork, work);
13931443
}),
13941444
);
1445+
it.effect("record reads select subagents by id", () => subagentRecordsSelectById);
13951446
it.effect(
13961447
"command reads skip finished leaf nodes of finished runs",
13971448
() => commandReadsSkipSettledLeafNodes,

‎apps/server/src/orchestration-v2/ProjectionStore.ts‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -333,6 +333,7 @@ export interface ProjectionRecordFilter {
333333
readonly messageRunIds?: ReadonlyArray<RunId>;
334334
readonly turnItemRunIds?: ReadonlyArray<RunId | null>;
335335
readonly runIds?: ReadonlyArray<RunId>;
336+
readonly subagentIds?: ReadonlyArray<NodeId>;
336337
readonly turnItemTypes?: ReadonlyArray<OrchestrationV2TurnItem["type"]>;
337338
readonly turnItemStatuses?: ReadonlyArray<OrchestrationV2TurnItem["status"]>;
338339
readonly omitSettledLeafNodes?: boolean;
@@ -3365,13 +3366,15 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
33653366
rows.toSorted((left, right) => left.position - right.position),
33663367
),
33673368
),
3368-
fields !== undefined && !fields.includes("subagents")
3369+
(fields !== undefined && !fields.includes("subagents")) ||
3370+
(window === undefined && matchesNothing(filter?.subagentIds))
33693371
? Effect.succeed([])
33703372
: window === undefined
33713373
? sql<PayloadRow>`
33723374
SELECT payload_json
33733375
FROM orchestration_v2_projection_subagents
33743376
WHERE thread_id = ${threadId}
3377+
${filter?.subagentIds === undefined ? sql`` : sql`AND subagent_id IN (SELECT value FROM json_each(${encodeIdList(filter.subagentIds)}))`}
33753378
ORDER BY COALESCE(started_at, ''), subagent_id ASC
33763379
`
33773380
: sql<PayloadRow>`
@@ -6828,7 +6831,7 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
68286831
getSettlementCandidates: readOnReader(getSettlementCandidates),
68296832
getThreadsWithPullRequests: readOnReader(getThreadsWithPullRequests),
68306833
getThreadWorktreePaths: readOnReader(getThreadWorktreePaths),
6831-
getOpenRemoteDelegatedTasks: readOnReader(getOpenRemoteDelegatedTasks),
6834+
getOpenRemoteDelegatedTasks: getOpenRemoteDelegatedTasks.pipe(NodeSqliteClient.readOnly),
68326835
getThreadProjection,
68336836
getTurnStartContext: readOnReader(getTurnStartContext),
68346837
getTurnStartHistory: readOnReader(getTurnStartHistory),
@@ -7278,6 +7281,10 @@ export const layerMemory: Layer.Layer<ProjectionStoreV2> = Layer.effect(
72787281
filter?.runIds === undefined
72797282
? projection.runs
72807283
: projection.runs.filter((row) => filter.runIds!.includes(row.id)),
7284+
subagents:
7285+
filter?.subagentIds === undefined
7286+
? projection.subagents
7287+
: projection.subagents.filter((row) => filter.subagentIds!.includes(row.id)),
72817288
turnItems: projection.turnItems.filter(
72827289
(row) =>
72837290
(filter?.turnItemRunIds === undefined ||

‎apps/server/src/orchestration-v2/RemoteDelegatedTask.test.ts‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -292,6 +292,56 @@ it.layer(layerTest)("a delegated task in a linked environment", (it) => {
292292
}),
293293
);
294294

295+
it.effect("owes a stop there for a Stop here that loses the race to its result", () =>
296+
Effect.gen(function* () {
297+
const orchestrator = yield* Orchestrator.OrchestratorV2;
298+
const projections = yield* ProjectionStore.ProjectionStoreV2;
299+
const parent = yield* seedLiveParent("stop-after-result");
300+
yield* orchestrator.dispatch({
301+
type: "delegated_task.remote.request",
302+
commandId: CommandId.make("command:remote-delegate:stop-after-result"),
303+
parentThreadId: parent.threadId,
304+
parentRunId: parent.runId,
305+
parentNodeId: parent.rootNodeId,
306+
task: "Run a check.",
307+
driver: ProviderDriverKind.make("remote"),
308+
modelSelection,
309+
remoteChild,
310+
});
311+
const task = (yield* orchestrator.getThreadProjection(parent.threadId)).subagents[0]!;
312+
const complete = {
313+
type: "delegated_task.remote.complete" as const,
314+
parentThreadId: parent.threadId,
315+
taskId: task.id,
316+
};
317+
yield* orchestrator.dispatch({
318+
...complete,
319+
commandId: CommandId.make("command:result:stop-after-result"),
320+
status: "completed",
321+
result: "Done there.",
322+
stoppedThere: "ran-out",
323+
});
324+
// The parent's Stop read the task before its result landed.
325+
yield* orchestrator.dispatch({
326+
...complete,
327+
commandId: CommandId.make("command:parent-stop:stop-after-result"),
328+
status: "interrupted",
329+
result: "Stopped with its parent.",
330+
});
331+
332+
const stopped = (yield* orchestrator.getThreadProjection(parent.threadId)).subagents[0]!;
333+
assert.equal(stopped.status, "completed");
334+
assert.equal(stopped.result, "Done there.");
335+
assert.equal(stopped.remoteChild?.stoppedThere, false);
336+
assert.equal(stopped.remoteChild?.stopCount, 1);
337+
const open = yield* projections.getOpenRemoteDelegatedTasks;
338+
assert.deepEqual(
339+
open.filter((entry) => entry.parentThreadId === parent.threadId),
340+
[{ parentThreadId: parent.threadId, taskId: task.id }],
341+
);
342+
}),
343+
);
344+
295345
it.effect("wakes its parent with the result when it completes there", () =>
296346
Effect.gen(function* () {
297347
const orchestrator = yield* Orchestrator.OrchestratorV2;
@@ -330,6 +380,7 @@ it.layer(layerTest)("a delegated task in a linked environment", (it) => {
330380
taskId: taskId!,
331381
status: "completed",
332382
result,
383+
stoppedThere: "ran-out",
333384
});
334385
yield* complete("All 412 tests pass on the box.");
335386
// A second report is refused, so the first result stands.

‎apps/server/src/orchestration-v2/ThreadManagementService.test.ts‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,12 +18,14 @@ import {
1818
} from "@t3tools/contracts";
1919
import * as Deferred from "effect/Deferred";
2020
import * as DateTime from "effect/DateTime";
21+
import * as Duration from "effect/Duration";
2122
import * as Option from "effect/Option";
2223
import * as Effect from "effect/Effect";
2324
import * as Fiber from "effect/Fiber";
2425
import * as Layer from "effect/Layer";
2526
import * as Queue from "effect/Queue";
2627
import * as Stream from "effect/Stream";
28+
import * as TestClock from "effect/testing/TestClock";
2729

2830
import * as ServerSettings from "../serverSettings.ts";
2931
import * as SqlitePersistence from "../persistence/Sqlite.ts";
@@ -385,6 +387,63 @@ it.effect("waitForThread can wait past an ended run for the thread's delegated w
385387
}),
386388
);
387389

390+
it.effect("waitForThread settles delegated work on a thread rolled back past its only run", () =>
391+
Effect.gen(function* () {
392+
const projectId = ProjectId.make("project:thread-management:wait-rolled-back");
393+
const threadId = ThreadId.make("thread:thread-management:wait-rolled-back");
394+
const runId = RunId.make("run:thread-management:wait-rolled-back");
395+
const rolledBack = {
396+
id: runId,
397+
ordinal: 1,
398+
status: "rolled_back",
399+
startedAt: DateTime.makeUnsafe(0),
400+
completedAt: DateTime.makeUnsafe(1_000),
401+
} as unknown as OrchestrationV2Run;
402+
const layerTest = ThreadManagementService.layer.pipe(
403+
Layer.provide(Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({})),
404+
Layer.provide(
405+
Layer.mock(Orchestrator.OrchestratorV2)({
406+
getThreadEventSequence: () => Effect.succeed(0),
407+
restartContinuationPending: () => Effect.succeed(false),
408+
delegatedTaskResultPending: () => Effect.succeed(false),
409+
streamStoredEventsFrom: () => Stream.never,
410+
getThreadRecords: () =>
411+
Effect.succeed({
412+
thread: { id: threadId, projectId, deletedAt: null },
413+
runs: [rolledBack],
414+
messages: [],
415+
subagents: [],
416+
providerThreads: [],
417+
} as unknown as OrchestrationV2ThreadProjection),
418+
}),
419+
),
420+
Layer.provide(ServerSettings.layerTest()),
421+
Layer.provide(layerReceiptsTest),
422+
);
423+
const service = yield* ThreadManagementService.ThreadManagementService.pipe(
424+
Effect.provide(layerTest),
425+
);
426+
const fiber = yield* service
427+
.waitForThread({
428+
projectId,
429+
threadId,
430+
runId,
431+
timeoutMs: 60_000,
432+
untilDelegatedWorkSettled: true,
433+
})
434+
.pipe(Effect.forkChild);
435+
// Nothing will ever run there again, so a wait still open runs out here.
436+
yield* TestClock.adjust(Duration.millis(60_000));
437+
const result = yield* Fiber.join(fiber);
438+
439+
expect(result).toMatchObject({
440+
timedOut: false,
441+
run: { id: runId, status: "rolled_back" },
442+
resultRun: { id: runId, status: "rolled_back" },
443+
});
444+
}),
445+
);
446+
388447
it.effect("waitForThread returns the cut run once its continuation is declined", () =>
389448
Effect.gen(function* () {
390449
const projectId = ProjectId.make("project:thread-management:wait-declined");

‎apps/server/src/orchestration-v2/ThreadManagementService.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -919,6 +919,13 @@ const make = Effect.gen(function* () {
919919
{ messageRoles: ["user"] },
920920
);
921921
const progress = delegatedTaskProgress(records);
922+
// A thread rolled back past all its work, with nothing left running, has no answer to wait for.
923+
if (
924+
progress.resultRun === undefined &&
925+
records.runs.every((run) => isTerminalRunStatus(run.status))
926+
) {
927+
return latestRun(records);
928+
}
922929
if (progress.state !== "result_available") return undefined;
923930
const pending = yield* orchestrator
924931
.delegatedTaskResultPending(input.threadId)

0 commit comments

Comments
 (0)