Skip to content

Commit 6c0e5cf

Browse files
fix(server): keep forwarded stop retries on their original work
1 parent 71cc1f0 commit 6c0e5cf

10 files changed

Lines changed: 378 additions & 239 deletions

File tree

‎apps/server/src/mcp/McpToolAccess.test.ts‎

Lines changed: 32 additions & 89 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,7 @@ import {
88
RuntimeMode,
99
ThreadId,
1010
} from "@t3tools/contracts";
11-
import * as Cause from "effect/Cause";
1211
import * as Effect from "effect/Effect";
13-
import * as Exit from "effect/Exit";
1412
import * as Layer from "effect/Layer";
1513
import * as Schema from "effect/Schema";
1614
import { McpSchema, McpServer, Tool, Toolkit } from "effect/ai";
@@ -350,38 +348,22 @@ it.effect("refuses a change when the calling thread cannot be read", () =>
350348
),
351349
);
352350

353-
const box = EnvironmentId.make("environment-box");
354-
const here = EnvironmentId.make("environment");
355-
const linkedProbe = {
356-
...probe,
357-
parameters: Schema.Struct({
358-
environmentId: OrchestratorMcpEnvironmentTarget,
359-
threadId: Schema.optional(ThreadId),
360-
runtimeMode: Schema.optional(RuntimeMode),
361-
interactionMode: Schema.optional(ProviderInteractionMode),
362-
}),
363-
};
364351
const LinkedProbeToolkit = Toolkit.make(
365-
Tool.make("reads", linkedProbe),
366-
Tool.make("writes", linkedProbe),
367-
Tool.make("writes_with_capability", linkedProbe),
368-
Tool.make("writes_threads", linkedProbe),
369-
Tool.make("starts_threads", linkedProbe),
370-
Tool.make("writes_environment", linkedProbe),
352+
Tool.make("writes_threads", {
353+
...probe,
354+
parameters: Schema.Struct({
355+
environmentId: OrchestratorMcpEnvironmentTarget,
356+
threadId: Schema.optional(ThreadId),
357+
}),
358+
}),
371359
);
372-
/** Runs here, and tells whether `environmentId` reached the handler. */
373-
const ranHere = <P extends object>(params: P) =>
374-
Effect.succeed({ ran: "environmentId" in params ? "saw environmentId" : "here" });
375-
376360
const linkedProbeServer = McpHttpServer.toolkitRegistration(
377361
LinkedProbeToolkit,
378362
McpToolAccess.toLayer(LinkedProbeToolkit, {
379-
reads: McpToolAccess.reads(ranHere),
380-
writes: McpToolAccess.writes(ranHere),
381-
writes_with_capability: McpToolAccess.writesWithCapability(ranHere),
382-
writes_threads: McpToolAccess.writesThreads((input) => [input.threadId], ranHere),
383-
starts_threads: McpToolAccess.startsThreads((input) => input, ranHere),
384-
writes_environment: McpToolAccess.writesEnvironment(ranHere),
363+
writes_threads: McpToolAccess.writesThreads(
364+
(input) => [input.threadId],
365+
(params) => Effect.succeed({ ran: "environmentId" in params ? "leaked target" : "here" }),
366+
),
385367
}),
386368
).pipe(
387369
Layer.provideMerge(McpServer.McpServer.layer),
@@ -390,71 +372,32 @@ const linkedProbeServer = McpHttpServer.toolkitRegistration(
390372
getThreadShell: (threadId) => Effect.succeed(shells.get(threadId) ?? null),
391373
}),
392374
),
393-
// Stands in for the linked environment, answering with where the call went.
394375
Layer.provide(
395376
Layer.mock(PeerForwarding.PeerForwarding)({
396-
call: (_scope, tool, environmentId) =>
397-
Effect.succeed({ ran: `${tool.name} in ${environmentId}` } as never),
377+
call: () => Effect.succeed({ ran: "there" } as never),
398378
}),
399379
),
400380
);
401381

402-
it.effect.each([
403-
["reads", supervised, { environmentId: box }, "reads in environment-box"],
404-
["reads", supervised, { environmentId: here }, "here"],
405-
["reads", supervised, {}, "here"],
406-
407-
// The caller is checked here before anything goes.
408-
["writes", readOnlyClient, { environmentId: box }, "capability_denied"],
409-
["writes", ended, { environmentId: box }, "parent_not_active"],
410-
["writes", supervised, { environmentId: box }, "writes in environment-box"],
411-
["writes_with_capability", readOnlyClient, { environmentId: box }, "capability_denied"],
412-
[
413-
"starts_threads",
414-
plan,
415-
{ environmentId: box, interactionMode: "default" },
416-
"interaction_mode_escalation_denied",
417-
],
418-
["starts_threads", plan, { environmentId: box }, "starts_threads in environment-box"],
419-
["writes_environment", supervised, { environmentId: box }, "capability_denied"],
420-
[
421-
"writes_environment",
422-
fullAccess,
423-
{ environmentId: box },
424-
"writes_environment in environment-box",
425-
],
426-
["writes_environment", fullAccess, { environmentId: here }, "here"],
427-
428-
// A thread id names a thread where the call runs, and is checked there.
429-
[
430-
"writes_threads",
431-
supervised,
432-
{ environmentId: box, threadId: fullAccessThreadId },
433-
"writes_threads in environment-box",
434-
],
435-
[
436-
"writes_threads",
437-
supervised,
438-
{ environmentId: here, threadId: fullAccessThreadId },
439-
"runtime_mode_escalation_denied",
440-
],
441-
] as const)("a linked %s from %o with %o: %s", ([tool, scope, args, expected]) =>
442-
call(tool, scope, args).pipe(
443-
Effect.map((outcome) => expect(outcome).toBe(expected)),
444-
Effect.provide(linkedProbeServer),
445-
),
446-
);
382+
it.effect(
383+
"checks the caller here and the target where it lives, even when thread IDs collide",
384+
() =>
385+
Effect.gen(function* () {
386+
const there = {
387+
environmentId: EnvironmentId.make("environment-box"),
388+
threadId: fullAccessThreadId,
389+
};
390+
expect(yield* call("writes_threads", supervised, there)).toBe("there");
391+
expect(yield* call("writes_threads", readOnlyClient, there)).toBe("capability_denied");
392+
expect(yield* call("writes_threads", ended, there)).toBe("parent_not_active");
447393

448-
it.effect("refuses a linked environment to a tool that acts as its calling thread", () =>
449-
Effect.gen(function* () {
450-
const CallerToolkit = Toolkit.make(Tool.make("acts_as_caller", linkedProbe));
451-
const built = yield* Layer.build(
452-
McpToolAccess.HandlersLayer.layer(
453-
McpToolAccess.toLayer(CallerToolkit, {
454-
acts_as_caller: McpToolAccess.actsAsCaller(ranHere),
455-
}),
456-
),
457-
).pipe(Effect.scoped, Effect.exit);
458-
expect(Exit.isFailure(built) && Cause.hasDies(built.cause)).toBe(true);
459-
}),
394+
const here = { ...there, environmentId: supervised.environmentId };
395+
expect(yield* call("writes_threads", supervised, here)).toBe(
396+
"runtime_mode_escalation_denied",
397+
);
398+
expect(yield* call("writes_threads", fullAccess, here)).toBe("here");
399+
expect(yield* call("writes_threads", fullAccess, { threadId: fullAccessThreadId })).toBe(
400+
"here",
401+
);
402+
}).pipe(Effect.provide(linkedProbeServer)),
460403
);

‎apps/server/src/mcp/OrchestratorMcpService.ts‎

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2713,11 +2713,21 @@ const make = Effect.gen(function* () {
27132713
yield* threadManagement
27142714
.stopDelegatedTasks({ threadId: input.threadId, commandId, reason: input.reason })
27152715
.pipe(Effect.catch(stopFailed));
2716-
// The run this stop interrupted, from its own events, so a retry
2717-
// reports the same run even if another has started since.
2718-
const interrupted = stopped.storedEvents.find(
2719-
(stored) => stored.event.type === "run.updated",
2720-
)?.event.runId;
2716+
// Queue holds and completion disposal also update runs. Only the
2717+
// original interrupt request identifies the run Stop reached.
2718+
const request = stopped.storedEvents.find(
2719+
(stored) =>
2720+
stored.event.type === "turn-item.updated" &&
2721+
stored.event.payload.type === "run_interrupt_request" &&
2722+
(stored.event.payload.providerTurnId !== null ||
2723+
stopped.storedEvents.some(
2724+
(updated) =>
2725+
updated.event.type === "run.updated" &&
2726+
updated.event.runId === stored.event.runId &&
2727+
updated.event.payload.status === "interrupted",
2728+
)),
2729+
);
2730+
const interrupted = request?.event.runId;
27212731
return {
27222732
threadId: input.threadId,
27232733
runId: interrupted ?? null,

‎apps/server/src/mcp/toolkits/project/handlers.ts‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@ import * as Project from "../../../project/ProjectService.ts";
1515
import * as ManagedProjectFolders from "../../../project/ManagedProjectFolders.ts";
1616
import * as Repositories from "../../../sourceControl/SourceControlRepositoryService.ts";
1717
import * as GitVcsDriver from "../../../vcs/GitVcsDriver.ts";
18-
import * as McpInvocationContext from "../../McpInvocationContext.ts";
1918
import * as McpToolAccess from "../../McpToolAccess.ts";
2019
import { newCommandId, readCaller, resolveProjectId, unavailable } from "../../threadAccess.ts";
2120
import { ProjectToolkit } from "./tools.ts";
@@ -62,9 +61,8 @@ export const layer = McpToolAccess.toLayer(ProjectToolkit, {
6261
(input) => input,
6362
(input, { runtimeMode, interactionMode }) =>
6463
Effect.gen(function* () {
65-
const scope = yield* McpInvocationContext.McpInvocationContext;
6664
const context = yield* readCaller();
67-
const { caller } = context;
65+
const { caller, scope } = context;
6866
// A retry with the same key replays the first launch: the ids derive
6967
// from the caller and the key, and the launch service replays a
7068
// command it already accepted.

‎apps/server/src/orchestration-v2/EventSink.ts‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -184,6 +184,11 @@ export interface EventSinkV2Shape {
184184
readonly latestSequence: (input?: {
185185
readonly threadId?: ThreadId;
186186
}) => Effect.Effect<number, EventSinkV2Error>;
187+
readonly read: (input: {
188+
readonly threadId: ThreadId;
189+
readonly throughSequence: number;
190+
readonly eventType: OrchestrationV2DomainEvent["type"];
191+
}) => Stream.Stream<OrchestrationV2StoredEvent, EventSinkStreamError>;
187192
readonly readByCommandId: (input: {
188193
readonly commandId: CommandId;
189194
}) => Stream.Stream<OrchestrationV2StoredEvent, EventSinkV2Error>;
@@ -901,6 +906,14 @@ const layerBase: Layer.Layer<
901906
}),
902907
),
903908
),
909+
read: (input) =>
910+
eventStore
911+
.read(input)
912+
.pipe(
913+
Stream.mapError(
914+
(cause) => new EventSinkStreamError({ threadId: input.threadId, cause }),
915+
),
916+
),
904917
readByCommandId: (input) =>
905918
eventStore.readByCommandId(input).pipe(
906919
Stream.mapError(

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -378,6 +378,12 @@ export interface OrchestratorV2Shape {
378378
readonly getThreadEventSequence: (
379379
threadId: ThreadId,
380380
) => Effect.Effect<number, OrchestratorV2Error>;
381+
/** Committed history at a fixed sequence, without subscribing to live events. */
382+
readonly readStoredEvents: (input: {
383+
readonly threadId: ThreadId;
384+
readonly throughSequence: number;
385+
readonly eventType: OrchestrationV2DomainEvent["type"];
386+
}) => Stream.Stream<OrchestrationV2StoredEvent, OrchestratorV2Error>;
381387
readonly streamStoredEvents: Stream.Stream<OrchestrationV2StoredEvent, OrchestratorV2Error>;
382388
readonly streamStoredEventsFrom: (input?: {
383389
readonly threadId?: ThreadId;
@@ -9717,6 +9723,37 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
97179723
),
97189724
);
97199725
const latestRun = projection.runs.findLast((run) => run.status !== "queued");
9726+
if (command.throughSequence !== undefined && latestRun !== undefined) {
9727+
// A child Stop may have failed before committing its receipt. Retrying
9728+
// it must not reach a run that started after the parent's original Stop.
9729+
const hadStarted = yield* eventSink
9730+
.read({
9731+
threadId: command.threadId,
9732+
throughSequence: command.throughSequence,
9733+
eventType: "run.created",
9734+
})
9735+
.pipe(
9736+
Stream.concat(
9737+
eventSink.read({
9738+
threadId: command.threadId,
9739+
throughSequence: command.throughSequence,
9740+
eventType: "run.updated",
9741+
}),
9742+
),
9743+
Stream.runFold(
9744+
() => false,
9745+
(found, stored) =>
9746+
found ||
9747+
((stored.event.type === "run.created" || stored.event.type === "run.updated") &&
9748+
stored.event.runId === latestRun.id &&
9749+
stored.event.payload.status !== "queued"),
9750+
),
9751+
Effect.mapError(
9752+
(cause) => new OrchestratorProjectionError({ threadId: command.threadId, cause }),
9753+
),
9754+
);
9755+
if (!hadStarted) return undefined;
9756+
}
97209757
const target =
97219758
projection.runs.findLast(
97229759
(run) =>
@@ -11772,6 +11809,10 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1177211809
eventSink
1177311810
.latestSequence({ threadId })
1177411811
.pipe(Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause }))),
11812+
readStoredEvents: (input) =>
11813+
eventSink
11814+
.read(input)
11815+
.pipe(Stream.mapError((cause) => new OrchestratorDomainEventStreamError({ cause }))),
1177511816
streamStoredEvents: eventSink.stream().pipe(
1177611817
Stream.mapError(
1177711818
(cause) =>
@@ -11926,6 +11967,12 @@ const layerUnavailable: Layer.Layer<OrchestratorV2> = Layer.succeed(
1192611967
cause: "Orchestration V2 live runtime is not configured.",
1192711968
}),
1192811969
),
11970+
readStoredEvents: () =>
11971+
Stream.fail(
11972+
new OrchestratorDomainEventStreamError({
11973+
cause: "Orchestration V2 live runtime is not configured.",
11974+
}),
11975+
),
1192911976
streamStoredEventsFrom: () =>
1193011977
Stream.fail(
1193111978
new OrchestratorDomainEventStreamError({

‎apps/server/src/orchestration-v2/ThreadManagementService.ts‎

Lines changed: 50 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ import * as Effect from "effect/Effect";
4343
import * as Layer from "effect/Layer";
4444
import * as Option from "effect/Option";
4545
import * as Ref from "effect/Ref";
46+
import * as Result from "effect/Result";
4647
import * as Schema from "effect/Schema";
4748
import * as Stream from "effect/Stream";
4849

@@ -1092,20 +1093,64 @@ const make = Effect.gen(function* () {
10921093

10931094
const stopDelegatedTasks: ThreadManagementServiceShape["stopDelegatedTasks"] = (input) =>
10941095
Effect.gen(function* () {
1095-
const { subagents } = yield* orchestrator.getThreadRecords(input.threadId, ["subagents"]);
1096+
const receipt = yield* receipts
1097+
.getByCommandId(input.commandId)
1098+
.pipe(
1099+
Effect.mapError(
1100+
(cause) =>
1101+
new Orchestrator.OrchestratorProjectionError({ threadId: input.threadId, cause }),
1102+
),
1103+
);
1104+
return yield* stopDelegatedTasksThrough(
1105+
input,
1106+
Option.isSome(receipt) ? receipt.value.resultSequence : undefined,
1107+
);
1108+
});
1109+
1110+
const stopDelegatedTasksThrough = (
1111+
input: Parameters<ThreadManagementServiceShape["stopDelegatedTasks"]>[0],
1112+
throughSequence: number | undefined,
1113+
): ReturnType<ThreadManagementServiceShape["stopDelegatedTasks"]> =>
1114+
Effect.gen(function* () {
1115+
// A retry must not stop tasks delegated after the original Stop, including
1116+
// when an earlier walk stopped some children and failed on another.
1117+
const childThreadIds = new Set(
1118+
throughSequence === undefined
1119+
? (yield* orchestrator.getThreadRecords(input.threadId, ["subagents"])).subagents.flatMap(
1120+
(task) =>
1121+
task.origin === "app_owned" && task.childThreadId !== null
1122+
? [task.childThreadId]
1123+
: [],
1124+
)
1125+
: yield* orchestrator
1126+
.readStoredEvents({
1127+
threadId: input.threadId,
1128+
throughSequence,
1129+
eventType: "subagent.updated",
1130+
})
1131+
.pipe(
1132+
Stream.filterMap((stored) =>
1133+
stored.event.type === "subagent.updated" &&
1134+
stored.event.payload.origin === "app_owned" &&
1135+
stored.event.payload.childThreadId !== null
1136+
? Result.succeed(stored.event.payload.childThreadId)
1137+
: Result.failVoid,
1138+
),
1139+
Stream.runCollect,
1140+
),
1141+
);
10961142
const failures: Array<
10971143
Orchestrator.OrchestratorV2Error | ThreadManagementProviderInstanceDisabledError
10981144
> = [];
1099-
for (const task of subagents) {
1100-
if (task.origin !== "app_owned" || task.childThreadId === null) continue;
1101-
const threadId = task.childThreadId;
1145+
for (const threadId of childThreadIds) {
11021146
yield* dispatch({
11031147
type: "thread.stop",
11041148
commandId: CommandId.make(`${input.commandId}:stop:${threadId}`),
11051149
threadId,
1150+
...(throughSequence === undefined ? {} : { throughSequence }),
11061151
...(input.reason === undefined ? {} : { reason: input.reason }),
11071152
}).pipe(
1108-
Effect.andThen(stopDelegatedTasks({ ...input, threadId })),
1153+
Effect.andThen(stopDelegatedTasksThrough({ ...input, threadId }, throughSequence)),
11091154
Effect.catch((error) =>
11101155
Effect.logWarning("Unable to stop a delegated task", {
11111156
parentThreadId: input.threadId,

0 commit comments

Comments
 (0)