Skip to content

Commit 5936612

Browse files
committed
fix(server): pressing Stop now clears orphaned delegated follow-ups
1 parent 1451e7c commit 5936612

2 files changed

Lines changed: 217 additions & 124 deletions

File tree

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 136 additions & 107 deletions
Original file line numberDiff line numberDiff line change
@@ -9594,6 +9594,121 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
95949594
return undefined;
95959595
});
95969596

9597+
const cancelIdleOrphans = (input: {
9598+
readonly command: Extract<
9599+
OrchestrationV2ServerCommand,
9600+
{ readonly type: "run.interrupt" | "thread.stop" }
9601+
>;
9602+
readonly events: Ref.Ref<Array<OrchestrationV2DomainEvent>>;
9603+
readonly projection: Pick<OrchestrationV2ThreadProjection, "subagents" | "turnItems">;
9604+
}) =>
9605+
Effect.gen(function* () {
9606+
const { command, events, projection } = input;
9607+
const pending = yield* Ref.get(events);
9608+
const children = new Map<
9609+
ThreadId,
9610+
ProjectionRecords<"runs" | "providerThreads" | "runtimeRequests" | "turnItems"> | null
9611+
>();
9612+
const idleChildren = new Map<ThreadId, boolean>();
9613+
let nodes: ReadonlyArray<OrchestrationV2ExecutionNode> | undefined;
9614+
for (const recorded of projection.subagents) {
9615+
const updated = pending.findLast(
9616+
(event) => event.type === "subagent.updated" && event.payload.id === recorded.id,
9617+
);
9618+
const task = updated?.type === "subagent.updated" ? updated.payload : recorded;
9619+
if (
9620+
task.origin !== "app_owned" ||
9621+
task.childThreadId === null ||
9622+
!isOrchestrationV2WorkActive(task.status)
9623+
)
9624+
continue;
9625+
const childThreadId = task.childThreadId;
9626+
const child = children.has(childThreadId)
9627+
? (children.get(childThreadId) ?? null)
9628+
: yield* projectionStore
9629+
.getThreadRecords(
9630+
childThreadId,
9631+
["runs", "providerThreads", "runtimeRequests", "turnItems"],
9632+
{
9633+
turnItemTypes: ["command_execution", "dynamic_tool", "subagent"],
9634+
turnItemStatuses: ["pending", "running", "waiting"],
9635+
},
9636+
)
9637+
.pipe(
9638+
Effect.catchTags({
9639+
ProjectionStoreThreadNotFoundError: () => Effect.succeed(null),
9640+
}),
9641+
Effect.mapError(
9642+
(cause) => new OrchestratorProjectionError({ threadId: childThreadId, cause }),
9643+
),
9644+
);
9645+
children.set(childThreadId, child);
9646+
if (child === null) continue;
9647+
if (
9648+
child.thread.lineage.parentThreadId !== command.threadId ||
9649+
child.thread.lineage.relationshipToParent !== "subagent" ||
9650+
(child.thread.forkedFrom?.type === "node" &&
9651+
child.thread.forkedFrom.nodeId === task.id) ||
9652+
child.runs.some((run) => run.delegatedTaskId === task.id)
9653+
)
9654+
continue;
9655+
const idle =
9656+
idleChildren.get(task.childThreadId) ??
9657+
(!child.runs.some((run) => run.status === "queued" || isBlockingRun(run)) &&
9658+
!child.providerThreads.some(
9659+
(thread) =>
9660+
thread.status === "active" || (thread.pendingBackgroundTasks?.length ?? 0) > 0,
9661+
) &&
9662+
!child.runtimeRequests.some((request) => request.status === "pending") &&
9663+
child.turnItems.length === 0 &&
9664+
Option.isNone(yield* Effect.findFirst(child.runs, awaitsRestartContinuation)));
9665+
idleChildren.set(task.childThreadId, idle);
9666+
if (!idle) continue;
9667+
// Stop can cancel an idle orphan without guessing which completed run owned its result.
9668+
const now = yield* DateTime.now;
9669+
const emitEvent = emit(events, command);
9670+
yield* emitEvent({
9671+
type: "subagent.updated",
9672+
threadId: command.threadId,
9673+
nodeId: task.id,
9674+
occurredAt: now,
9675+
payload: {
9676+
...task,
9677+
status: "cancelled",
9678+
completedAt: now,
9679+
updatedAt: now,
9680+
completionDelivery: { state: "disposed", observedByRunId: null },
9681+
},
9682+
});
9683+
nodes ??= (yield* projectionStore.getThreadRecords(command.threadId, ["nodes"])).nodes;
9684+
const node = nodes.find((candidate) => candidate.id === task.id);
9685+
if (node !== undefined)
9686+
yield* emitEvent({
9687+
type: "node.updated",
9688+
threadId: command.threadId,
9689+
nodeId: node.id,
9690+
occurredAt: now,
9691+
payload: { ...node, status: "cancelled", completedAt: now },
9692+
});
9693+
for (const item of projection.turnItems) {
9694+
if (item.type !== "subagent" || item.subagentId !== task.id) continue;
9695+
yield* emitEvent({
9696+
type: "turn-item.updated",
9697+
threadId: command.threadId,
9698+
nodeId: task.id,
9699+
occurredAt: now,
9700+
payload: { ...item, status: "cancelled", completedAt: now, updatedAt: now },
9701+
});
9702+
}
9703+
}
9704+
}).pipe(
9705+
Effect.mapError((cause) =>
9706+
Schema.is(OrchestratorProjectionError)(cause)
9707+
? cause
9708+
: new OrchestratorProjectionError({ threadId: input.command.threadId, cause }),
9709+
),
9710+
);
9711+
95979712
const dispatchThreadStop = (
95989713
command: Extract<OrchestrationV2ServerCommand, { readonly type: "thread.stop" }>,
95999714
events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
@@ -9611,111 +9726,6 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
96119726
),
96129727
);
96139728
if (projection.thread.deletedAt !== null) return undefined;
9614-
const cancelIdleOrphans = Effect.gen(function* () {
9615-
const pending = yield* Ref.get(events);
9616-
const children = new Map<
9617-
ThreadId,
9618-
ProjectionRecords<"runs" | "providerThreads" | "runtimeRequests" | "turnItems"> | null
9619-
>();
9620-
const idleChildren = new Map<ThreadId, boolean>();
9621-
let nodes: ReadonlyArray<OrchestrationV2ExecutionNode> | undefined;
9622-
for (const recorded of projection.subagents) {
9623-
const updated = pending.findLast(
9624-
(event) => event.type === "subagent.updated" && event.payload.id === recorded.id,
9625-
);
9626-
const task = updated?.type === "subagent.updated" ? updated.payload : recorded;
9627-
if (
9628-
task.origin !== "app_owned" ||
9629-
task.childThreadId === null ||
9630-
!isOrchestrationV2WorkActive(task.status)
9631-
)
9632-
continue;
9633-
const childThreadId = task.childThreadId;
9634-
const child = children.has(childThreadId)
9635-
? (children.get(childThreadId) ?? null)
9636-
: yield* projectionStore
9637-
.getThreadRecords(
9638-
childThreadId,
9639-
["runs", "providerThreads", "runtimeRequests", "turnItems"],
9640-
{
9641-
turnItemTypes: ["command_execution", "dynamic_tool", "subagent"],
9642-
turnItemStatuses: ["pending", "running", "waiting"],
9643-
},
9644-
)
9645-
.pipe(
9646-
Effect.catchTags({
9647-
ProjectionStoreThreadNotFoundError: () => Effect.succeed(null),
9648-
}),
9649-
Effect.mapError(
9650-
(cause) => new OrchestratorProjectionError({ threadId: childThreadId, cause }),
9651-
),
9652-
);
9653-
children.set(childThreadId, child);
9654-
if (child === null) continue;
9655-
if (
9656-
child.thread.lineage.parentThreadId !== command.threadId ||
9657-
child.thread.lineage.relationshipToParent !== "subagent" ||
9658-
(child.thread.forkedFrom?.type === "node" &&
9659-
child.thread.forkedFrom.nodeId === task.id) ||
9660-
child.runs.some((run) => run.delegatedTaskId === task.id)
9661-
)
9662-
continue;
9663-
const idle =
9664-
idleChildren.get(task.childThreadId) ??
9665-
(!child.runs.some((run) => run.status === "queued" || isBlockingRun(run)) &&
9666-
!child.providerThreads.some(
9667-
(thread) =>
9668-
thread.status === "active" || (thread.pendingBackgroundTasks?.length ?? 0) > 0,
9669-
) &&
9670-
!child.runtimeRequests.some((request) => request.status === "pending") &&
9671-
child.turnItems.length === 0 &&
9672-
Option.isNone(yield* Effect.findFirst(child.runs, awaitsRestartContinuation)));
9673-
idleChildren.set(task.childThreadId, idle);
9674-
if (!idle) continue;
9675-
// Stop can cancel an idle orphan without guessing which completed run owned its result.
9676-
const now = yield* DateTime.now;
9677-
const emitEvent = emit(events, command);
9678-
yield* emitEvent({
9679-
type: "subagent.updated",
9680-
threadId: command.threadId,
9681-
nodeId: task.id,
9682-
occurredAt: now,
9683-
payload: {
9684-
...task,
9685-
status: "cancelled",
9686-
completedAt: now,
9687-
updatedAt: now,
9688-
completionDelivery: { state: "disposed", observedByRunId: null },
9689-
},
9690-
});
9691-
nodes ??= (yield* projectionStore.getThreadRecords(command.threadId, ["nodes"])).nodes;
9692-
const node = nodes.find((candidate) => candidate.id === task.id);
9693-
if (node !== undefined)
9694-
yield* emitEvent({
9695-
type: "node.updated",
9696-
threadId: command.threadId,
9697-
nodeId: node.id,
9698-
occurredAt: now,
9699-
payload: { ...node, status: "cancelled", completedAt: now },
9700-
});
9701-
for (const item of projection.turnItems) {
9702-
if (item.type !== "subagent" || item.subagentId !== task.id) continue;
9703-
yield* emitEvent({
9704-
type: "turn-item.updated",
9705-
threadId: command.threadId,
9706-
nodeId: task.id,
9707-
occurredAt: now,
9708-
payload: { ...item, status: "cancelled", completedAt: now, updatedAt: now },
9709-
});
9710-
}
9711-
}
9712-
}).pipe(
9713-
Effect.mapError((cause) =>
9714-
Schema.is(OrchestratorProjectionError)(cause)
9715-
? cause
9716-
: new OrchestratorProjectionError({ threadId: command.threadId, cause }),
9717-
),
9718-
);
97199729
const latestRun = projection.runs.findLast((run) => run.status !== "queued");
97209730
const target =
97219731
projection.runs.findLast(
@@ -9754,7 +9764,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
97549764
const enqueued = yield* Ref.get(interruptEffects);
97559765
yield* Ref.update(events, (existing) => [...existing, ...written]);
97569766
yield* Ref.update(effects, (existing) => [...existing, ...enqueued]);
9757-
yield* cancelIdleOrphans;
9767+
yield* cancelIdleOrphans({ command, events, projection });
97589768
return interrupted.value;
97599769
}
97609770
yield* Effect.logWarning("thread.stop could not interrupt the running turn", {
@@ -9775,7 +9785,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
97759785
yield* markStoppedRun({ command, events, thread: projection.thread, run, now });
97769786
}
97779787
yield* holdStoppedThread({ command, events, projection, cohortRunIds: [], now });
9778-
yield* cancelIdleOrphans;
9788+
yield* cancelIdleOrphans({ command, events, projection });
97799789
return undefined;
97809790
});
97819791

@@ -11100,6 +11110,25 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1110011110
case "run.interrupt":
1110111111
cancelUnsettledEffects = yield* dispatchRunInterrupt(command, events, effects);
1110211112
if (command.holdQueue === true) {
11113+
// Stop must still commit when a child can't be read, so a failed repair's
11114+
// partial events are dropped.
11115+
const repairEvents = yield* Ref.make(yield* Ref.get(events));
11116+
const repaired = yield* Effect.exit(
11117+
loadProjectionForCommand(command, ["subagents", "turnItems"], {
11118+
turnItemTypes: ["subagent"],
11119+
turnItemStatuses: ["pending", "running", "waiting"],
11120+
}).pipe(
11121+
Effect.flatMap((projection) =>
11122+
cancelIdleOrphans({ command, events: repairEvents, projection }),
11123+
),
11124+
),
11125+
);
11126+
if (Exit.isSuccess(repaired)) yield* Ref.set(events, yield* Ref.get(repairEvents));
11127+
else
11128+
yield* Effect.logWarning("Stop could not cancel orphaned delegated follow-ups", {
11129+
threadId: command.threadId,
11130+
cause: repaired.cause,
11131+
});
1110311132
yield* Ref.update(effects, (existing) => [
1110411133
...existing,
1110511134
{

0 commit comments

Comments
 (0)