Skip to content

Commit 5108c97

Browse files
authored
fix(server): restarts keep delegated tasks, queued threads, and stops intact (#15323)
1 parent f786ff3 commit 5108c97

29 files changed

Lines changed: 2494 additions & 172 deletions

‎apps/server/src/mcp/OrchestratorMcpService.test.ts‎

Lines changed: 101 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,12 @@ import {
1111
type OrchestrationV2ThreadProjection,
1212
type ServerProvider,
1313
} from "@t3tools/contracts";
14+
import * as DateTime from "effect/DateTime";
1415
import * as Effect from "effect/Effect";
1516
import * as Layer from "effect/Layer";
1617
import * as Ref from "effect/Ref";
1718

19+
import { OrchestratorProjectionError } from "../orchestration-v2/Orchestrator.ts";
1820
import type { ProviderAdapterV2Shape } from "../orchestration-v2/ProviderAdapter.ts";
1921
import * as ProviderAdapterRegistry from "../orchestration-v2/ProviderAdapterRegistry.ts";
2022
import * as ThreadManagementService from "../orchestration-v2/ThreadManagementService.ts";
@@ -52,7 +54,22 @@ describe("OrchestratorMcpService", () => {
5254
} as unknown as OrchestrationV2ThreadProjection;
5355
const childProjection = {
5456
thread: { id: childThreadId },
55-
runs: [{ id: childRunId, ordinal: 1, status: "completed" }],
57+
runs: [
58+
{
59+
id: childRunId,
60+
ordinal: 1,
61+
status: "completed",
62+
startedAt: DateTime.makeUnsafe("2026-10-03T10:00:00Z"),
63+
completedAt: DateTime.makeUnsafe("2026-10-03T10:10:00Z"),
64+
},
65+
{
66+
id: RunId.make("run:mcp-ack-continuation"),
67+
ordinal: 2,
68+
status: "failed",
69+
startedAt: DateTime.makeUnsafe("2026-10-03T10:02:00Z"),
70+
completedAt: DateTime.makeUnsafe("2026-10-03T10:05:00Z"),
71+
},
72+
],
5673
contextTransfers: [],
5774
messages: [],
5875
subagents: [],
@@ -126,13 +143,96 @@ describe("OrchestratorMcpService", () => {
126143
const result = yield* service.taskStatus(scope, taskId);
127144
assert.equal(result.status, "completed");
128145
assert.equal(result.summary, "terminal result");
146+
assert.equal(result.latestTerminalRunId, childRunId);
147+
assert.equal(result.latestTerminalStatus, "completed");
129148
const commandIds = yield* Ref.get(acknowledgementCommandIds);
130149
assert.equal(commandIds.length, 2);
131150
assert.notEqual(commandIds[0], commandIds[1]);
132151
}).pipe(Effect.provide(OrchestratorMcpService.layer.pipe(Layer.provide(dependencies))));
133152
}),
134153
);
135154

155+
it.effect("reports a restart-cut child as working until its continuation settles", () =>
156+
Effect.gen(function* () {
157+
const parentThreadId = ThreadId.make("thread:mcp-restart-parent");
158+
const childThreadId = ThreadId.make("thread:mcp-restart-child");
159+
const taskId = NodeId.make("node:mcp-restart-task");
160+
const dispatched = yield* Ref.make(0);
161+
let awaitsRestart = true;
162+
let readFails = true;
163+
const parentProjection = {
164+
thread: { id: parentThreadId },
165+
runs: [],
166+
contextTransfers: [],
167+
subagents: [
168+
{
169+
id: taskId,
170+
threadId: parentThreadId,
171+
origin: "app_owned",
172+
childThreadId,
173+
driver: "codex",
174+
model: "gpt-5.6-terra",
175+
status: "running",
176+
result: null,
177+
completionDelivery: { state: "pending" },
178+
},
179+
],
180+
} as unknown as OrchestrationV2ThreadProjection;
181+
const childProjection = {
182+
thread: { id: childThreadId },
183+
runs: [{ id: RunId.make("run:mcp-restart-child"), ordinal: 1, status: "cancelled" }],
184+
contextTransfers: [],
185+
messages: [],
186+
subagents: [],
187+
providerThreads: [],
188+
turnItems: [],
189+
} as unknown as OrchestrationV2ThreadProjection;
190+
const dependencies = Layer.mergeAll(
191+
NodeServices.layer,
192+
Layer.mock(ThreadManagementService.ThreadManagementService)({
193+
getThreadRecords: (threadId) =>
194+
Effect.succeed(threadId === parentThreadId ? parentProjection : childProjection),
195+
delegatedTaskResultPending: () =>
196+
readFails
197+
? Effect.fail(new OrchestratorProjectionError({ threadId: childThreadId }))
198+
: Effect.succeed(awaitsRestart),
199+
dispatch: () => Ref.update(dispatched, (count) => count + 1).pipe(Effect.as({} as never)),
200+
}),
201+
Layer.mock(ProviderRegistry.ProviderRegistry)({ getProviders: Effect.succeed([]) }),
202+
Layer.mock(ProviderAdapterRegistry.ProviderAdapterRegistryV2)({
203+
list: () => Effect.succeed([]),
204+
}),
205+
Layer.mock(ScheduledTaskService.ScheduledTaskService)({}),
206+
);
207+
const scope: McpInvocationScope = {
208+
environmentId: EnvironmentId.make("environment:mcp-restart"),
209+
threadId: parentThreadId,
210+
providerSessionId: "provider-session:mcp-restart",
211+
providerInstanceId: ProviderInstanceId.make("codex"),
212+
capabilities: new Set(["orchestration"]),
213+
issuedAt: 1,
214+
};
215+
216+
yield* Effect.gen(function* () {
217+
const service = yield* OrchestratorMcpService.OrchestratorMcpService;
218+
const failed = yield* service.taskStatus(scope, taskId).pipe(Effect.flip);
219+
assert.equal(failed.code, "orchestration_error");
220+
assert.equal(yield* Ref.get(dispatched), 0);
221+
readFails = false;
222+
const held = yield* service.taskStatus(scope, taskId);
223+
assert.equal(held.status, "running");
224+
assert.equal(held.workState, "working");
225+
assert.isNull(held.summary);
226+
// Acknowledging the cut run would suppress the real result's wake.
227+
assert.equal(yield* Ref.get(dispatched), 0);
228+
awaitsRestart = false;
229+
const settled = yield* service.taskStatus(scope, taskId);
230+
assert.equal(settled.status, "cancelled");
231+
assert.equal(yield* Ref.get(dispatched), 1);
232+
}).pipe(Effect.provide(OrchestratorMcpService.layer.pipe(Layer.provide(dependencies))));
233+
}),
234+
);
235+
136236
it.effect("does not dispose delivery when a nonterminal task has no active child run", () =>
137237
Effect.gen(function* () {
138238
const parentThreadId = ThreadId.make("thread:mcp-cancel-parent");

‎apps/server/src/mcp/OrchestratorMcpService.ts‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ import {
5252
type ServerProvider,
5353
ThreadId,
5454
} from "@t3tools/contracts";
55+
import { runRanAfter } from "@t3tools/shared/orchestrationV2ThreadError";
5556
import * as Context from "effect/Context";
5657
import * as Crypto from "effect/Crypto";
5758
import * as DateTime from "effect/DateTime";
@@ -412,7 +413,10 @@ function latestTerminalResultRun(
412413
run.status !== "rolled_back" &&
413414
(run.id === delegatedRun?.id || run.startedAt !== null),
414415
)
415-
.toSorted((left, right) => right.ordinal - left.ordinal)[0];
416+
.reduce<OrchestrationV2Run | undefined>(
417+
(latest, run) => (latest === undefined || runRanAfter(run, latest) ? run : latest),
418+
undefined,
419+
);
416420
}
417421

418422
function canExposeTaskRunResult(run: OrchestrationV2Run | undefined): run is OrchestrationV2Run {
@@ -1055,7 +1059,16 @@ const make = Effect.gen(function* () {
10551059
messages: [...childControls.messages, ...resultRecords.messages],
10561060
turnItems: resultRecords.turnItems,
10571061
};
1058-
const workState = task.result !== null ? "result_available" : progress.state;
1062+
// A restart cut the child's run and its continuation has not settled, or
1063+
// the child started working again after this read.
1064+
const heldForRestart =
1065+
task.result === null &&
1066+
progress.state === "result_available" &&
1067+
(yield* threadManagement
1068+
.delegatedTaskResultPending(task.childThreadId)
1069+
.pipe(Effect.mapError(threadManagementFailure)));
1070+
const workState =
1071+
task.result !== null ? "result_available" : heldForRestart ? "working" : progress.state;
10591072
const status =
10601073
task.result !== null
10611074
? taskStatusForRun(

0 commit comments

Comments
 (0)