Skip to content

Commit f1fa11e

Browse files
committed
fix(orchestration): reconcile held delegated work without stale running state
1 parent 31a9da1 commit f1fa11e

15 files changed

Lines changed: 1405 additions & 163 deletions
Lines changed: 234 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,234 @@
1+
import * as NodeServices from "@effect/platform-node/NodeServices";
2+
import { assert, it } from "@effect/vitest";
3+
import {
4+
EnvironmentId,
5+
MessageId,
6+
NodeId,
7+
type OrchestrationV2ThreadProjection,
8+
type OrchestrationV2ServerCommand,
9+
ProjectId,
10+
ProviderInstanceId,
11+
RunId,
12+
ThreadId,
13+
} from "@t3tools/contracts";
14+
import * as DateTime from "effect/DateTime";
15+
import * as Effect from "effect/Effect";
16+
import * as Layer from "effect/Layer";
17+
18+
import * as ProviderAdapterRegistry from "../orchestration-v2/ProviderAdapterRegistry.ts";
19+
import * as ThreadManagementService from "../orchestration-v2/ThreadManagementService.ts";
20+
import * as ProviderRegistry from "../provider/Services/ProviderRegistry.ts";
21+
import * as ScheduledTaskService from "../scheduledTasks/ScheduledTaskService.ts";
22+
import type { McpInvocationScope } from "./McpInvocationContext.ts";
23+
import * as OrchestratorMcpService from "./OrchestratorMcpService.ts";
24+
25+
const parentThreadId = ThreadId.make("thread:queued-task-parent");
26+
const childThreadId = ThreadId.make("thread:queued-task-child");
27+
const taskId = NodeId.make("node:queued-task");
28+
const now = DateTime.makeUnsafe("2026-10-03T11:00:00Z");
29+
const providerInstanceId = ProviderInstanceId.make("claude_two");
30+
const scope: McpInvocationScope = {
31+
environmentId: EnvironmentId.make("environment:queued-task"),
32+
threadId: parentThreadId,
33+
providerSessionId: "provider-session:queued-task",
34+
providerInstanceId,
35+
capabilities: new Set(["orchestration"]),
36+
issuedAt: 1,
37+
};
38+
39+
function fixture(
40+
options: {
41+
published?: boolean;
42+
executing?: boolean;
43+
failCancellation?: boolean;
44+
secondQueued?: boolean;
45+
deliveryPending?: boolean;
46+
} = {},
47+
) {
48+
const projectId = ProjectId.make("project:queued-task");
49+
const thread = {
50+
id: childThreadId,
51+
projectId,
52+
title: "Child",
53+
modelSelection: { instanceId: providerInstanceId, model: "claude-opus-5-5" },
54+
lineage: { parentThreadId, relationshipToParent: "subagent", rootThreadId: parentThreadId },
55+
createdAt: now,
56+
updatedAt: now,
57+
deletedAt: null,
58+
archivedAt: null,
59+
settledOverride: null,
60+
settledAt: null,
61+
runtimeMode: "full-access",
62+
interactionMode: "default",
63+
};
64+
const run = (ordinal: number, status: string) => ({
65+
id: RunId.make(`run:queued-task:${ordinal}`),
66+
threadId: childThreadId,
67+
ordinal,
68+
status,
69+
userMessageId: MessageId.make(`message:queued-task:${ordinal}`),
70+
requestedAt: now,
71+
startedAt: status === "queued" ? null : now,
72+
completedAt: status === "failed" || status === "completed" ? now : null,
73+
queueHeld: status === "queued",
74+
modelSelection: thread.modelSelection,
75+
providerInstanceId,
76+
});
77+
const child = {
78+
thread,
79+
runs: [
80+
run(1, "failed"),
81+
run(2, "queued"),
82+
run(3, "completed"),
83+
run(4, "completed"),
84+
...(options.executing ? [run(5, "running")] : []),
85+
...(options.secondQueued ? [run(6, "queued")] : []),
86+
],
87+
messages: [
88+
{
89+
id: MessageId.make("result:queued-task:4"),
90+
runId: RunId.make("run:queued-task:4"),
91+
role: "assistant",
92+
text: "Latest completed follow-up result",
93+
updatedAt: now,
94+
},
95+
],
96+
turnItems: [],
97+
providerThreads: [{ status: "idle", pendingBackgroundTasks: [] }],
98+
runtimeRequests: [],
99+
contextTransfers: [],
100+
subagents: [],
101+
} as unknown as OrchestrationV2ThreadProjection;
102+
const parent = {
103+
thread: {
104+
...thread,
105+
id: parentThreadId,
106+
lineage: { parentThreadId: null, relationshipToParent: null },
107+
},
108+
runs: [],
109+
contextTransfers: [],
110+
subagents: [
111+
{
112+
id: taskId,
113+
threadId: parentThreadId,
114+
childThreadId,
115+
origin: "app_owned",
116+
status: options.published ? "completed" : "running",
117+
providerInstanceId,
118+
model: "claude-opus-5-5",
119+
result: options.published ? "Published original result" : null,
120+
completionDelivery: { state: options.deliveryPending ? "pending" : "disposed" },
121+
},
122+
],
123+
} as unknown as OrchestrationV2ThreadProjection;
124+
const commands: OrchestrationV2ServerCommand[] = [];
125+
const layer = OrchestratorMcpService.layer.pipe(
126+
Layer.provide(
127+
Layer.mergeAll(
128+
NodeServices.layer,
129+
Layer.mock(ThreadManagementService.ThreadManagementService)({
130+
getThreadRecords: (id) => Effect.succeed(id === parentThreadId ? parent : child),
131+
getProjectThreadRecords: () => Effect.succeed(child),
132+
getTimelinePage: () => Effect.succeed({ items: [], totalItems: 0, hasMore: false }),
133+
dispatch: (command) =>
134+
Effect.suspend(() => {
135+
commands.push(command);
136+
return options.failCancellation && command.type === "queued-run.cancel"
137+
? Effect.fail(new Error("Cancellation failed") as never)
138+
: Effect.succeed({} as never);
139+
}),
140+
}),
141+
Layer.mock(ProviderRegistry.ProviderRegistry)({ getProviders: Effect.succeed([]) }),
142+
Layer.mock(ProviderAdapterRegistry.ProviderAdapterRegistryV2)({
143+
list: () => Effect.succeed([]),
144+
}),
145+
Layer.mock(ScheduledTaskService.ScheduledTaskService)({}),
146+
),
147+
),
148+
);
149+
return { child, parent, commands, layer };
150+
}
151+
152+
it.effect("reports an old held continuation as queued and keeps the latest result readable", () => {
153+
const { commands, layer } = fixture();
154+
return Effect.gen(function* () {
155+
const service = yield* OrchestratorMcpService.OrchestratorMcpService;
156+
const task = yield* service.taskStatus(scope, taskId);
157+
assert.equal(task.status, "queued");
158+
assert.equal(task.workState, "working");
159+
assert.equal(task.hasPendingChildRuns, true);
160+
assert.equal(task.latestTerminalRunId, RunId.make("run:queued-task:4"));
161+
assert.equal(task.latestTerminalSummary, "Latest completed follow-up result");
162+
assert.isNull(task.summary);
163+
const read = yield* service.readThread(scope, { threadId: childThreadId, runLimit: 1 });
164+
assert.equal(read.thread.status, "completed");
165+
assert.isNull(read.thread.activeRunId);
166+
assert.equal(read.thread.pendingRequestCount, 0);
167+
assert.equal(read.thread.queuedRunCount, 1);
168+
assert.equal(read.thread.heldQueuedRunCount, 1);
169+
assert.equal(read.recentRuns.length, 1);
170+
assert.deepEqual(commands, []);
171+
}).pipe(Effect.provide(layer));
172+
});
173+
174+
it.effect("cancels queued-only unpublished work without a provider interruption", () => {
175+
const { commands, child, layer } = fixture({ secondQueued: true });
176+
return Effect.gen(function* () {
177+
const service = yield* OrchestratorMcpService.OrchestratorMcpService;
178+
const result = yield* service.cancelTask(scope, { taskId, clientRequestId: "cancel-held" });
179+
assert.equal(result.status, "cancel_requested");
180+
assert.deepEqual(
181+
commands.map((command) => command.type),
182+
["queued-run.cancel"],
183+
);
184+
const command = commands[0]!;
185+
assert.equal(command.type, "queued-run.cancel");
186+
if (command.type !== "queued-run.cancel")
187+
return yield* Effect.die("Expected queue cancellation");
188+
assert.equal(command.threadId, childThreadId);
189+
assert.equal(command.runId, child.runs[1]!.id);
190+
assert.deepEqual(command.additionalRunIds, [RunId.make("run:queued-task:6")]);
191+
}).pipe(Effect.provide(layer));
192+
});
193+
194+
it.effect("keeps truly executing child work running ahead of held queued work", () => {
195+
const { layer } = fixture({ executing: true });
196+
return Effect.gen(function* () {
197+
const service = yield* OrchestratorMcpService.OrchestratorMcpService;
198+
const result = yield* service.taskStatus(scope, taskId);
199+
assert.equal(result.status, "running");
200+
assert.equal(result.hasPendingChildRuns, true);
201+
}).pipe(Effect.provide(layer));
202+
});
203+
204+
it.effect("does not resurrect a published task or cancel its later backing-thread work", () => {
205+
const { commands, layer } = fixture({ published: true, executing: true });
206+
return Effect.gen(function* () {
207+
const service = yield* OrchestratorMcpService.OrchestratorMcpService;
208+
const status = yield* service.taskStatus(scope, taskId);
209+
assert.equal(status.status, "completed");
210+
assert.equal(status.summary, "Published original result");
211+
assert.equal(status.hasPendingChildRuns, true);
212+
const cancel = yield* service.cancelTask(scope, {
213+
taskId,
214+
clientRequestId: "cancel-published",
215+
});
216+
assert.equal(cancel.status, "completed");
217+
assert.deepEqual(commands, []);
218+
}).pipe(Effect.provide(layer));
219+
});
220+
221+
it.effect("does not dispose completion delivery when queue cancellation fails", () => {
222+
const { commands, layer } = fixture({ failCancellation: true, deliveryPending: true });
223+
return Effect.gen(function* () {
224+
const service = yield* OrchestratorMcpService.OrchestratorMcpService;
225+
const error = yield* service
226+
.cancelTask(scope, { taskId, clientRequestId: "cancel-fails" })
227+
.pipe(Effect.flip);
228+
assert.equal(error.code, "task_not_cancellable");
229+
assert.deepEqual(
230+
commands.map((command) => command.type),
231+
["queued-run.cancel"],
232+
);
233+
}).pipe(Effect.provide(layer));
234+
});

‎apps/server/src/mcp/OrchestratorMcpService.ts‎

Lines changed: 57 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -624,6 +624,10 @@ function threadDetail(
624624
pendingRequestCount: projection.runtimeRequests.filter(
625625
(request) => request.status === "pending",
626626
).length,
627+
queuedRunCount: projection.runs.filter((run) => run.status === "queued").length,
628+
heldQueuedRunCount: projection.runs.filter(
629+
(run) => run.status === "queued" && run.queueHeld === true,
630+
).length,
627631
archived: projection.thread.archivedAt !== null,
628632
...threadSettlement(projection.thread),
629633
createdAt: DateTime.formatIso(projection.thread.createdAt),
@@ -1068,9 +1072,11 @@ const make = Effect.gen(function* () {
10681072
)
10691073
: workState === "result_available"
10701074
? taskStatusForRun(progress.resultRun ?? childRun)
1071-
: taskStatusForRun(childRun) === "queued"
1072-
? "queued"
1073-
: "running";
1075+
: progress.pendingRun !== undefined
1076+
? taskStatusForRun(progress.pendingRun)
1077+
: taskStatusForRun(childRun) === "queued"
1078+
? "queued"
1079+
: "running";
10741080
const derivedResult =
10751081
task.result !== null
10761082
? task.result
@@ -1521,32 +1527,59 @@ const make = Effect.gen(function* () {
15211527
}
15221528
const child = yield* loadProjection(current.childThreadId);
15231529
const activeRun = ThreadManagementService.latestActiveRun(child);
1524-
if (activeRun === undefined) {
1530+
const queuedRuns = child.runs.filter((run) => run.status === "queued");
1531+
if (activeRun === undefined && queuedRuns.length === 0) {
15251532
return yield* failure(
15261533
"task_not_cancellable",
1527-
`Delegated task ${input.taskId} has no interruptible child run.`,
1534+
`Delegated task ${input.taskId} has no interruptible or queued child run.`,
15281535
);
15291536
}
1530-
yield* threadManagement
1531-
.dispatch({
1532-
type: "run.interrupt",
1533-
commandId: stableCommandId({
1534-
scope,
1535-
requestKey: key,
1536-
operation: "cancel-task",
1537-
}),
1538-
threadId: current.childThreadId,
1539-
runId: activeRun.id,
1540-
...(input.reason === undefined ? {} : { reason: input.reason }),
1541-
})
1542-
.pipe(
1543-
Effect.mapError((error) =>
1544-
failure(
1545-
"task_not_cancellable",
1546-
`Unable to interrupt delegated task ${input.taskId}: ${errorMessage(error)}`,
1537+
// Cancel unstarted intent atomically so terminal reactions cannot
1538+
// promote a continuation that this cancellation also owns.
1539+
const firstQueuedRun = queuedRuns[0];
1540+
if (firstQueuedRun !== undefined) {
1541+
yield* threadManagement
1542+
.dispatch({
1543+
type: "queued-run.cancel",
1544+
commandId: stableCommandId({
1545+
scope,
1546+
requestKey: key,
1547+
operation: "cancel-task-queued",
1548+
}),
1549+
threadId: current.childThreadId,
1550+
runId: firstQueuedRun.id,
1551+
additionalRunIds: queuedRuns.slice(1).map((run) => run.id),
1552+
})
1553+
.pipe(
1554+
Effect.mapError((error) =>
1555+
failure(
1556+
"task_not_cancellable",
1557+
`Unable to cancel queued runs for delegated task ${input.taskId}: ${errorMessage(error)}`,
1558+
),
15471559
),
1548-
),
1549-
);
1560+
);
1561+
}
1562+
if (activeRun !== undefined)
1563+
yield* threadManagement
1564+
.dispatch({
1565+
type: "run.interrupt",
1566+
commandId: stableCommandId({
1567+
scope,
1568+
requestKey: key,
1569+
operation: "cancel-task",
1570+
}),
1571+
threadId: current.childThreadId,
1572+
runId: activeRun.id,
1573+
...(input.reason === undefined ? {} : { reason: input.reason }),
1574+
})
1575+
.pipe(
1576+
Effect.mapError((error) =>
1577+
failure(
1578+
"task_not_cancellable",
1579+
`Unable to interrupt delegated task ${input.taskId}: ${errorMessage(error)}`,
1580+
),
1581+
),
1582+
);
15501583
yield* disposeCompletionDelivery.pipe(
15511584
Effect.catchCause((cause) =>
15521585
Effect.logWarning("orchestrator-mcp.cancel-task.delivery-dispose-failed", {

‎apps/server/src/mcp/toolkits/orchestrator/tools.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ const TaskStatusTool = Tool.make("task_status", {
8484

8585
const TaskCancelTool = Tool.make("task_cancel", {
8686
description:
87-
"Request interruption of an active T3-owned delegated task and dispose its automatic parent delivery. For a terminal task, return its existing status and dispose delivery without interrupting later child-thread runs, even when task_status reports hasPendingChildRuns=true. Published task results remain available. Use t3_thread_interrupt for a later active run.",
87+
"Cancel queued continuations and request interruption of an active, unpublished T3-owned delegated task, then dispose its automatic parent delivery. A queued-only task can be cancelled without a live provider run. For a terminal task, return its existing status and dispose delivery without interrupting later child-thread runs, even when task_status reports hasPendingChildRuns=true. Published task results remain available. Use t3_thread_interrupt for a later active run.",
8888
parameters: OrchestratorMcpTaskCancelInput,
8989
success: OrchestratorMcpTaskCancelResult,
9090
failure: OrchestratorMcpFailure,
@@ -173,7 +173,7 @@ const ThreadListTool = Tool.make("t3_thread_list", {
173173

174174
const ThreadReadTool = Tool.make("t3_thread_read", {
175175
description:
176-
"Read durable state and a paginated timeline from a T3 thread in the calling project, or from a thread the user attached to this conversation as context. The default messages view returns user messages, assistant messages, and proposed plans; activity returns all summarized timeline items. Reading an untruncated terminal assistant result from this parent thread's direct app-owned child acknowledges that child's automatic completion delivery. Continue with afterPosition=nextPosition. Recover long item text with itemId and textOffset=nextTextOffset until nextTextOffset is null; offsets count UTF-16 code units.",
176+
"Read durable state and a paginated timeline from a T3 thread in the calling project, or from a thread the user attached to this conversation as context. The default messages view returns user messages, assistant messages, and proposed plans; activity returns all summarized timeline items. pendingRequestCount counts provider approval/input requests; queuedRunCount and heldQueuedRunCount report unstarted message runs independently of the latest run status. Reading an untruncated terminal assistant result from this parent thread's direct app-owned child acknowledges that child's automatic completion delivery. Continue with afterPosition=nextPosition. Recover long item text with itemId and textOffset=nextTextOffset until nextTextOffset is null; offsets count UTF-16 code units.",
177177
parameters: OrchestratorMcpThreadReadInput,
178178
success: OrchestratorMcpThreadReadResult,
179179
failure: OrchestratorMcpFailure,

0 commit comments

Comments
 (0)