Skip to content

Commit d307ede

Browse files
committed
fix(server): hold delegated results through repeated restarts and failed continuations
- Recovery orders runs by when they ended, so a queued run resumed after a continuation is still continued, and its background roster run is loaded. - A child stays held while any later continuation is pending, task_status reports it as working, and a continuation that fails for good settles it. - settled_only siblings join a delivery once their spawning run ended. - The replay harness runs delegated task recovery like startup does, and held-queue selection tolerates malformed payloads.
1 parent 4f7f58c commit d307ede

15 files changed

Lines changed: 452 additions & 18 deletions

‎apps/server/src/mcp/OrchestratorMcpService.test.ts‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,79 @@ describe("OrchestratorMcpService", () => {
133133
}),
134134
);
135135

136+
it.effect("reports a restart-cut child as working until its continuation settles", () =>
137+
Effect.gen(function* () {
138+
const parentThreadId = ThreadId.make("thread:mcp-restart-parent");
139+
const childThreadId = ThreadId.make("thread:mcp-restart-child");
140+
const taskId = NodeId.make("node:mcp-restart-task");
141+
const dispatched = yield* Ref.make(0);
142+
let awaitsRestart = true;
143+
const parentProjection = {
144+
thread: { id: parentThreadId },
145+
runs: [],
146+
contextTransfers: [],
147+
subagents: [
148+
{
149+
id: taskId,
150+
threadId: parentThreadId,
151+
origin: "app_owned",
152+
childThreadId,
153+
driver: "codex",
154+
model: "gpt-5.6-terra",
155+
status: "running",
156+
result: null,
157+
completionDelivery: { state: "pending" },
158+
},
159+
],
160+
} as unknown as OrchestrationV2ThreadProjection;
161+
const childProjection = {
162+
thread: { id: childThreadId },
163+
runs: [{ id: RunId.make("run:mcp-restart-child"), ordinal: 1, status: "cancelled" }],
164+
contextTransfers: [],
165+
messages: [],
166+
subagents: [],
167+
providerThreads: [],
168+
turnItems: [],
169+
} as unknown as OrchestrationV2ThreadProjection;
170+
const dependencies = Layer.mergeAll(
171+
NodeServices.layer,
172+
Layer.mock(ThreadManagementService.ThreadManagementService)({
173+
getThreadRecords: (threadId) =>
174+
Effect.succeed(threadId === parentThreadId ? parentProjection : childProjection),
175+
delegatedTaskAwaitsRestart: () => Effect.sync(() => awaitsRestart),
176+
dispatch: () => Ref.update(dispatched, (count) => count + 1).pipe(Effect.as({} as never)),
177+
}),
178+
Layer.mock(ProviderRegistry.ProviderRegistry)({ getProviders: Effect.succeed([]) }),
179+
Layer.mock(ProviderAdapterRegistry.ProviderAdapterRegistryV2)({
180+
list: () => Effect.succeed([]),
181+
}),
182+
Layer.mock(ScheduledTaskService.ScheduledTaskService)({}),
183+
);
184+
const scope: McpInvocationScope = {
185+
environmentId: EnvironmentId.make("environment:mcp-restart"),
186+
threadId: parentThreadId,
187+
providerSessionId: "provider-session:mcp-restart",
188+
providerInstanceId: ProviderInstanceId.make("codex"),
189+
capabilities: new Set(["orchestration"]),
190+
issuedAt: 1,
191+
};
192+
193+
yield* Effect.gen(function* () {
194+
const service = yield* OrchestratorMcpService.OrchestratorMcpService;
195+
const held = yield* service.taskStatus(scope, taskId);
196+
assert.equal(held.status, "running");
197+
assert.equal(held.workState, "working");
198+
assert.isNull(held.summary);
199+
// Acknowledging the cut run would suppress the real result's wake.
200+
assert.equal(yield* Ref.get(dispatched), 0);
201+
awaitsRestart = false;
202+
const settled = yield* service.taskStatus(scope, taskId);
203+
assert.equal(settled.status, "cancelled");
204+
assert.equal(yield* Ref.get(dispatched), 1);
205+
}).pipe(Effect.provide(OrchestratorMcpService.layer.pipe(Layer.provide(dependencies))));
206+
}),
207+
);
208+
136209
it.effect("does not dispose delivery when a nonterminal task has no active child run", () =>
137210
Effect.gen(function* () {
138211
const parentThreadId = ThreadId.make("thread:mcp-cancel-parent");

‎apps/server/src/mcp/OrchestratorMcpService.ts‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1055,7 +1055,13 @@ const make = Effect.gen(function* () {
10551055
messages: [...childControls.messages, ...resultRecords.messages],
10561056
turnItems: resultRecords.turnItems,
10571057
};
1058-
const workState = task.result !== null ? "result_available" : progress.state;
1058+
// A restart cut the child's run and its continuation has not settled yet.
1059+
const heldForRestart =
1060+
task.result === null &&
1061+
progress.state === "result_available" &&
1062+
(yield* threadManagement.delegatedTaskAwaitsRestart(task.childThreadId));
1063+
const workState =
1064+
task.result !== null ? "result_available" : heldForRestart ? "working" : progress.state;
10591065
const status =
10601066
task.result !== null
10611067
? taskStatusForRun(

‎apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts‎

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -386,6 +386,98 @@ it.layer(TestLayer)("delegated completion delivery repairs", (it) => {
386386
}),
387387
);
388388

389+
it.effect("acceptance batches a settled_only sibling once its spawning run ended", () =>
390+
Effect.gen(function* () {
391+
const orchestrator = yield* Orchestrator.OrchestratorV2;
392+
const sink = yield* EventSink.EventSinkV2;
393+
const now = yield* DateTime.now;
394+
const threadId = ThreadId.make("mailbox-mixed");
395+
const runId = RunId.make("mailbox-mixed-parent");
396+
const taskId = NodeId.make("mailbox-mixed-always");
397+
const siblingId = NodeId.make("mailbox-mixed-settled-only");
398+
const messageId = MessageId.make(`message:delegated-delivery:${threadId}`);
399+
yield* seedParentWithTerminalTask({
400+
threadId,
401+
runId,
402+
projectId: ProjectId.make("mailbox-mixed-project"),
403+
rootNodeId: NodeId.make("mailbox-mixed-root"),
404+
taskId,
405+
deliveryState: "claimed",
406+
completionWake: "always",
407+
deliveryTaskIds: [taskId],
408+
now,
409+
});
410+
const projection = yield* orchestrator.getThreadProjection(threadId);
411+
const task = projection.subagents[0]!;
412+
const spawningRun = projection.runs.find((row) => row.id === runId)!;
413+
// A restart cut the spawning run; its continuation is the live run now.
414+
yield* sink.write({
415+
events: [
416+
{
417+
...runEvent({ threadId, runId, ordinal: 1, status: "cancelled", now }),
418+
payload: { ...spawningRun, status: "cancelled" as const, completedAt: now },
419+
},
420+
runEvent({
421+
threadId,
422+
runId: RunId.make("mailbox-mixed-continuation"),
423+
ordinal: 2,
424+
status: "running",
425+
now,
426+
}),
427+
{
428+
id: EventId.make("mailbox-mixed-message"),
429+
type: "message.updated",
430+
threadId,
431+
runId,
432+
occurredAt: now,
433+
payload: {
434+
id: messageId,
435+
threadId,
436+
runId,
437+
nodeId: task.parentNodeId,
438+
role: "user",
439+
text: "Background task finished",
440+
attachments: [],
441+
streaming: false,
442+
createdBy: "agent",
443+
creationSource: "server",
444+
createdAt: now,
445+
updatedAt: now,
446+
delegatedCompletion: { parentRunId: runId, generation: 1, taskIds: [taskId] },
447+
},
448+
},
449+
{
450+
id: EventId.make(`event:${siblingId}`),
451+
type: "subagent.updated" as const,
452+
threadId,
453+
runId,
454+
nodeId: siblingId,
455+
occurredAt: now,
456+
payload: {
457+
...task,
458+
id: siblingId,
459+
completionWake: "settled_only" as const,
460+
completionDelivery: { state: "pending" as const, observedByRunId: null },
461+
},
462+
},
463+
],
464+
});
465+
yield* orchestrator.dispatch({
466+
type: "notification.delivery.accept",
467+
commandId: CommandId.make("accept-mixed"),
468+
threadId,
469+
messageId,
470+
});
471+
const accepted = yield* orchestrator.getThreadProjection(threadId);
472+
const cohort = accepted.runs.find((row) => row.id === runId)?.delegatedCompletion;
473+
assert.deepEqual(cohort?.delivery?.taskIds, [siblingId]);
474+
assert.equal(
475+
accepted.subagents.find((row) => row.id === siblingId)?.completionDelivery?.state,
476+
"claimed",
477+
);
478+
}),
479+
);
480+
389481
it.effect("builds completion text and metadata from the same live cohort", () =>
390482
Effect.gen(function* () {
391483
const orchestrator = yield* Orchestrator.OrchestratorV2;
@@ -852,6 +944,38 @@ it.layer(TestLayer)("delegated tasks across a server restart", (it) => {
852944
const stopped = yield* child("restart-stopped-child", false);
853945
// Settled with only background work left: its interim reply is not the result.
854946
const backgrounded = yield* child("restart-backgrounded-child", true, "completed");
947+
// A second restart cut the first continuation before it started.
948+
const recut = yield* child("restart-recut-child", false);
949+
const recutContinuationId = RunId.make("run:restart-recut-child:2");
950+
const recutCommandId = CommandId.make("command:restart-recut-child:reconcile");
951+
const recutRun = runEvent({
952+
threadId: recut.childThreadId,
953+
runId: recutContinuationId,
954+
ordinal: 2,
955+
status: "cancelled",
956+
now,
957+
});
958+
yield* eventSink.writeWithEffects({
959+
commandId: recutCommandId,
960+
events: [
961+
{
962+
...recutRun,
963+
payload: {
964+
...recutRun.payload,
965+
startedAt: null,
966+
restartContinuationOfRunId: recut.childRunId,
967+
},
968+
},
969+
],
970+
effects: [
971+
{
972+
id: `effect:restart-continuation:${recutContinuationId}`,
973+
commandId: recutCommandId,
974+
threadId: recut.childThreadId,
975+
request: { type: "provider-runtime.continue", sourceRunId: recutContinuationId },
976+
},
977+
],
978+
});
855979

856980
yield* orchestrator.recoverDelegatedTasks;
857981

@@ -863,6 +987,10 @@ it.layer(TestLayer)("delegated tasks across a server restart", (it) => {
863987
assert.isNull(task(resumed.taskId)?.result ?? null);
864988
assert.equal(task(backgrounded.taskId)?.status, "running");
865989
assert.isNull(task(backgrounded.taskId)?.result ?? null);
990+
assert.equal(task(recut.taskId)?.status, "running");
991+
assert.isTrue(yield* orchestrator.delegatedTaskAwaitsRestart(recut.childThreadId));
992+
assert.isTrue(yield* orchestrator.delegatedTaskAwaitsRestart(resumed.childThreadId));
993+
assert.isFalse(yield* orchestrator.delegatedTaskAwaitsRestart(stopped.childThreadId));
866994
assert.isFalse(
867995
recovered.contextTransfers.some(
868996
(transfer) => transfer.sourceThreadId === resumed.childThreadId,

‎apps/server/src/orchestration-v2/EffectWorker.test.ts‎

Lines changed: 49 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,8 @@ function restartEffect(
7979
function makeExecutorLayer(input: {
8080
readonly events: Ref.Ref<ReadonlyArray<string>>;
8181
readonly failFirstStart?: Ref.Ref<boolean>;
82+
readonly threads?: Partial<ThreadManagementService.ThreadManagementService["Service"]>;
83+
readonly continueAfterRestart?: boolean;
8284
}) {
8385
const record = (event: string) => Ref.update(input.events, (events) => [...events, event]);
8486
const dependencies = Layer.mergeAll(
@@ -148,8 +150,10 @@ function makeExecutorLayer(input: {
148150
Layer.provide(
149151
Layer.mergeAll(
150152
dependencies,
151-
Layer.mock(ThreadManagementService.ThreadManagementService)({}),
152-
ServerSettings.layerTest(),
153+
Layer.mock(ThreadManagementService.ThreadManagementService)(input.threads ?? {}),
154+
ServerSettings.layerTest(
155+
input.continueAfterRestart === true ? { continueThreadsAfterServerUpdate: true } : {},
156+
),
153157
),
154158
),
155159
);
@@ -753,3 +757,46 @@ it.effect("safely retries after replacement cleanup succeeds and start fails", (
753757
]);
754758
}),
755759
);
760+
761+
it.effect("settles a delegated child once its restart continuation fails for good", () =>
762+
Effect.gen(function* () {
763+
const timestamp = DateTime.formatIso(yield* DateTime.now);
764+
const events = yield* Ref.make<ReadonlyArray<string>>([]);
765+
const recovered = yield* Ref.make<ReadonlyArray<ThreadId>>([]);
766+
const layer = makeExecutorLayer({
767+
events,
768+
continueAfterRestart: true,
769+
threads: {
770+
getThreadRecords: () => Effect.fail(new Error("provider instance removed") as never),
771+
recoverDelegatedTask: (childThreadId) =>
772+
Ref.update(recovered, (ids) => [...ids, childThreadId]),
773+
},
774+
});
775+
const effect: EffectOutbox.OrchestrationEffectV2 = {
776+
id: `effect:restart-continuation:${runId}`,
777+
commandId: CommandId.make("command:restart-continuation-failure"),
778+
threadId,
779+
request: { type: "provider-runtime.continue", sourceRunId: runId },
780+
status: "running",
781+
attemptCount: 1,
782+
availableAt: timestamp,
783+
leaseOwner: "test-worker",
784+
leaseExpiresAt: timestamp,
785+
createdAt: timestamp,
786+
updatedAt: timestamp,
787+
completedAt: null,
788+
lastError: null,
789+
};
790+
yield* Effect.gen(function* () {
791+
const executor = yield* EffectWorker.OrchestrationEffectExecutorV2;
792+
assert.isTrue(
793+
Exit.isFailure(yield* Effect.exit(executor.execute(effect, { willRetry: true }))),
794+
);
795+
assert.deepEqual(yield* Ref.get(recovered), []);
796+
assert.isTrue(
797+
Exit.isFailure(yield* Effect.exit(executor.execute(effect, { willRetry: false }))),
798+
);
799+
assert.deepEqual(yield* Ref.get(recovered), [threadId]);
800+
}).pipe(Effect.provide(layer));
801+
}),
802+
);

‎apps/server/src/orchestration-v2/EffectWorker.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,10 @@ export const executorLayer: Layer.Layer<
115115
}).pipe(
116116
Effect.provideService(ThreadManagementService.ThreadManagementService, threads),
117117
Effect.provideService(ServerSettings.ServerSettingsService, settings),
118+
// A continuation that will never run still owes a delegated parent a result.
119+
Effect.tapError(() =>
120+
willRetry ? Effect.void : threads.recoverDelegatedTask(effect.threadId),
121+
),
118122
Effect.mapError(
119123
(cause) =>
120124
new OrchestrationEffectExecutionError({

0 commit comments

Comments
 (0)