Skip to content

Commit cee6d05

Browse files
khush-2106claude
andcommitted
fix(server): Stop ends a run whose turn its adapter already settled
When a provider adapter has already settled a turn but the run never projected that turn's end, Stop did nothing: the adapter's interrupt returns without a terminal for a turn it no longer tracks, so the run, attempt and provider turn stayed running and the thread looked busy until a restart. The Stop interrupt now waits for a running turn's terminal to project, as restart already does. The settle follow-up then ends the stopped run as interrupted if its live attempt's turn is still running, since no terminal will ever arrive for it. Refs #15197 Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1 parent 70e5a40 commit cee6d05

3 files changed

Lines changed: 366 additions & 29 deletions

File tree

‎apps/server/src/orchestration-v2/Orchestrator.control-reads.test.ts‎

Lines changed: 180 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -534,3 +534,183 @@ it.effect("settles only the stopped run's background work, once", () =>
534534
]);
535535
}).pipe(Effect.provide(testLayer)),
536536
);
537+
538+
// Stop on a turn its adapter had already settled: the adapter reports nothing
539+
// and the run never projected the turn's end (#15197). The settle follow-up
540+
// ends that run, but not one whose live attempt superseded the stopped turn.
541+
it.effect("ends a stopped run whose settled turn never projected its end", () =>
542+
Effect.gen(function* () {
543+
const orchestrator = yield* Orchestrator.OrchestratorV2;
544+
const projections = yield* ProjectionStore.ProjectionStoreV2;
545+
const now = yield* DateTime.now;
546+
const seedRunningRun = (name: string, options: { readonly steered: boolean }) =>
547+
Effect.gen(function* () {
548+
const threadId = ThreadId.make(`thread:${name}`);
549+
const providerThreadId = ProviderThreadId.make(`provider-thread:${name}`);
550+
const runId = RunId.make(`run:${name}`);
551+
const attemptId = RunAttemptId.make(`attempt:${name}`);
552+
const nodeId = NodeId.make(`node:${name}`);
553+
const providerTurnId = ProviderTurnId.make(`provider-turn:${name}`);
554+
yield* orchestrator.dispatch({
555+
type: "thread.create",
556+
commandId: CommandId.make(`create-${name}`),
557+
threadId,
558+
projectId: ProjectId.make(`project:${name}`),
559+
title: name,
560+
modelSelection,
561+
runtimeMode: "full-access",
562+
interactionMode: "default",
563+
branch: null,
564+
worktreePath: null,
565+
createdBy: "user",
566+
creationSource: "web",
567+
});
568+
yield* projections.apply({
569+
id: EventId.make(`${name}:provider-thread`),
570+
type: "provider-thread.updated",
571+
threadId,
572+
occurredAt: now,
573+
payload: {
574+
id: providerThreadId,
575+
driver: adapter.driver,
576+
providerInstanceId: instanceId,
577+
providerSessionId: null,
578+
appThreadId: threadId,
579+
ownerNodeId: null,
580+
nativeThreadRef: null,
581+
nativeConversationHeadRef: null,
582+
status: "active",
583+
firstRunOrdinal: 1,
584+
lastRunOrdinal: 1,
585+
handoffIds: [],
586+
forkedFrom: null,
587+
createdAt: now,
588+
updatedAt: now,
589+
},
590+
});
591+
yield* projections.apply({
592+
id: EventId.make(`${name}:run`),
593+
type: "run.created",
594+
threadId,
595+
occurredAt: now,
596+
payload: {
597+
id: runId,
598+
threadId,
599+
ordinal: 1,
600+
providerInstanceId: instanceId,
601+
modelSelection,
602+
providerThreadId,
603+
userMessageId: MessageId.make(`message:${name}`),
604+
rootNodeId: nodeId,
605+
activeAttemptId: options.steered
606+
? RunAttemptId.make(`attempt:${name}:steer`)
607+
: attemptId,
608+
status: "running",
609+
requestedAt: now,
610+
startedAt: now,
611+
completedAt: null,
612+
checkpointId: null,
613+
contextHandoffId: null,
614+
},
615+
});
616+
yield* projections.apply({
617+
id: EventId.make(`${name}:node`),
618+
type: "node.updated",
619+
threadId,
620+
runId,
621+
nodeId,
622+
occurredAt: now,
623+
payload: {
624+
id: nodeId,
625+
threadId,
626+
runId,
627+
parentNodeId: null,
628+
rootNodeId: nodeId,
629+
kind: "root_turn",
630+
status: "running",
631+
countsForRun: true,
632+
providerThreadId,
633+
providerTurnId,
634+
nativeItemRef: null,
635+
runtimeRequestId: null,
636+
checkpointScopeId: null,
637+
startedAt: now,
638+
completedAt: null,
639+
},
640+
});
641+
yield* projections.apply({
642+
id: EventId.make(`${name}:attempt`),
643+
type: "run-attempt.created",
644+
threadId,
645+
occurredAt: now,
646+
payload: {
647+
id: attemptId,
648+
runId,
649+
attemptOrdinal: 1,
650+
rootNodeId: nodeId,
651+
providerInstanceId: instanceId,
652+
providerThreadId,
653+
providerTurnId,
654+
reason: "initial",
655+
status: "running",
656+
startedAt: now,
657+
completedAt: null,
658+
},
659+
});
660+
yield* projections.apply({
661+
id: EventId.make(`${name}:turn`),
662+
type: "provider-turn.updated",
663+
threadId,
664+
occurredAt: now,
665+
payload: {
666+
id: providerTurnId,
667+
providerThreadId,
668+
nodeId,
669+
runAttemptId: attemptId,
670+
nativeTurnRef: null,
671+
ordinal: 1,
672+
status: "running",
673+
startedAt: now,
674+
completedAt: null,
675+
},
676+
});
677+
yield* orchestrator.dispatch({
678+
type: "thread.background-work.settle",
679+
commandId: CommandId.make(`stop-${name}:background-work-settled`),
680+
threadId,
681+
providerThreadId,
682+
providerTurnId,
683+
});
684+
const projection = yield* projections.getThreadProjection(threadId);
685+
return {
686+
run: projection.runs.find((run) => run.id === runId)?.status,
687+
attempt: projection.attempts.find((attempt) => attempt.id === attemptId)?.status,
688+
providerTurn: projection.providerTurns.find((turn) => turn.id === providerTurnId)?.status,
689+
rootNode: projection.nodes.find((node) => node.id === nodeId)?.status,
690+
providerThread: projection.providerThreads.find(
691+
(thread) => thread.id === providerThreadId,
692+
)?.status,
693+
interruptResult: projection.turnItems.some(
694+
(item) => item.runId === runId && item.type === "run_interrupt_result",
695+
),
696+
};
697+
});
698+
699+
assert.deepEqual(yield* seedRunningRun("orphaned-stop", { steered: false }), {
700+
run: "interrupted",
701+
attempt: "interrupted",
702+
providerTurn: "interrupted",
703+
rootNode: "interrupted",
704+
providerThread: "idle",
705+
interruptResult: true,
706+
});
707+
assert.deepEqual(yield* seedRunningRun("steered-stop", { steered: true }), {
708+
run: "running",
709+
attempt: "running",
710+
providerTurn: "running",
711+
rootNode: "running",
712+
providerThread: "active",
713+
interruptResult: false,
714+
});
715+
}).pipe(Effect.provide(testLayer)),
716+
);

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 137 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -7846,16 +7846,38 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
78467846
(cause) => new OrchestratorProjectionError({ threadId: command.threadId, cause }),
78477847
),
78487848
);
7849-
const stoppedRunId = projection.attempts.find(
7850-
(attempt) => attempt.id === stopped.providerTurn?.runAttemptId,
7851-
)?.runId;
7852-
const stoppedRun = projection.runs.find((run) => run.id === stoppedRunId);
7849+
const stoppedTurn = stopped.providerTurn;
7850+
const stoppedAttempt = projection.attempts.find(
7851+
(attempt) => attempt.id === stoppedTurn?.runAttemptId,
7852+
);
7853+
const stoppedRun = projection.runs.find((run) => run.id === stoppedAttempt?.runId);
7854+
if (stoppedRun === undefined || stoppedAttempt === undefined || stoppedTurn === undefined) {
7855+
return;
7856+
}
7857+
const now = yield* DateTime.now;
7858+
// The interrupt waited for this turn's terminal to project. A turn still
7859+
// running here was already settled by its adapter, which had nothing to
7860+
// stop and reported nothing, so no terminal will ever end this run.
7861+
const orphaned =
7862+
stoppedTurn.status === "running" &&
7863+
stoppedRun.status === "running" &&
7864+
stoppedRun.activeAttemptId === stoppedAttempt.id;
7865+
if (orphaned) {
7866+
yield* interruptOrphanedRun({
7867+
command,
7868+
events,
7869+
run: stoppedRun,
7870+
attempt: stoppedAttempt,
7871+
providerTurn: stoppedTurn,
7872+
now,
7873+
});
7874+
}
78537875
// A new turn may have started since Stop; its work is not this Stop's.
78547876
if (
7855-
stoppedRun === undefined ||
78567877
projection.runs.some(
78577878
(run) =>
7858-
run.status === "preparing" || run.status === "starting" || run.status === "running",
7879+
!(orphaned && run.id === stoppedRun.id) &&
7880+
(run.status === "preparing" || run.status === "starting" || run.status === "running"),
78597881
)
78607882
) {
78617883
return;
@@ -7866,8 +7888,116 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
78667888
projection,
78677889
stoppedProviderThreadId: command.providerThreadId,
78687890
throughRunOrdinal: stoppedRun.ordinal,
7869-
now: yield* DateTime.now,
7891+
now,
7892+
});
7893+
});
7894+
7895+
/**
7896+
* Ends a run as Stop would have, for a turn whose terminal its adapter had
7897+
* already emitted but the run never projected. Matches the terminal events
7898+
* RunExecutionService writes for an interrupted turn.
7899+
*/
7900+
const interruptOrphanedRun = (input: {
7901+
readonly command: Extract<
7902+
OrchestrationV2InternalCommand,
7903+
{ readonly type: "thread.background-work.settle" }
7904+
>;
7905+
readonly events: Ref.Ref<Array<OrchestrationV2DomainEvent>>;
7906+
readonly run: OrchestrationV2Run;
7907+
readonly attempt: OrchestrationV2RunAttempt;
7908+
readonly providerTurn: OrchestrationV2ProviderTurn;
7909+
readonly now: DateTime.Utc;
7910+
}) =>
7911+
Effect.gen(function* () {
7912+
const { run, attempt, providerTurn, now } = input;
7913+
const records = yield* projectionStore
7914+
.getThreadRecords(run.threadId, ["nodes", "providerThreads"])
7915+
.pipe(
7916+
Effect.mapError(
7917+
(cause) => new OrchestratorProjectionError({ threadId: run.threadId, cause }),
7918+
),
7919+
);
7920+
const rootNode = records.nodes.find((node) => node.id === run.rootNodeId);
7921+
const providerThread = records.providerThreads.find(
7922+
(candidate) => candidate.id === providerTurn.providerThreadId,
7923+
);
7924+
const emitEvent = emit(input.events, input.command);
7925+
yield* emitEvent({
7926+
type: "provider-turn.updated",
7927+
threadId: run.threadId,
7928+
nodeId: providerTurn.nodeId,
7929+
providerInstanceId: run.providerInstanceId,
7930+
occurredAt: now,
7931+
payload: { ...providerTurn, status: "interrupted", completedAt: now },
7932+
});
7933+
yield* emitEvent({
7934+
type: "run-attempt.updated",
7935+
threadId: run.threadId,
7936+
runId: run.id,
7937+
nodeId: attempt.rootNodeId,
7938+
providerInstanceId: run.providerInstanceId,
7939+
occurredAt: now,
7940+
payload: { ...attempt, status: "interrupted", completedAt: now },
78707941
});
7942+
yield* emitEvent({
7943+
type: "turn-item.updated",
7944+
threadId: run.threadId,
7945+
runId: run.id,
7946+
nodeId: attempt.rootNodeId,
7947+
providerInstanceId: run.providerInstanceId,
7948+
occurredAt: now,
7949+
payload: {
7950+
id: idAllocator.derive.runSignalTurnItem({ runId: run.id, signal: "interrupt-result" }),
7951+
threadId: run.threadId,
7952+
runId: run.id,
7953+
nodeId: attempt.rootNodeId,
7954+
providerThreadId: providerTurn.providerThreadId,
7955+
providerTurnId: providerTurn.id,
7956+
nativeItemRef: null,
7957+
parentItemId: idAllocator.derive.runSignalTurnItem({
7958+
runId: run.id,
7959+
signal: "interrupt-request",
7960+
}),
7961+
ordinal: run.ordinal * 100 + 98,
7962+
status: "interrupted",
7963+
title: "Interrupted",
7964+
startedAt: now,
7965+
completedAt: now,
7966+
updatedAt: now,
7967+
type: "run_interrupt_result",
7968+
message: "Run interrupted by user",
7969+
},
7970+
});
7971+
yield* emitEvent({
7972+
type: "run.updated",
7973+
threadId: run.threadId,
7974+
runId: run.id,
7975+
nodeId: attempt.rootNodeId,
7976+
providerInstanceId: run.providerInstanceId,
7977+
occurredAt: now,
7978+
payload: { ...run, status: "interrupted", completedAt: now },
7979+
});
7980+
if (rootNode !== undefined) {
7981+
yield* emitEvent({
7982+
type: "node.updated",
7983+
threadId: run.threadId,
7984+
runId: run.id,
7985+
nodeId: rootNode.id,
7986+
providerInstanceId: run.providerInstanceId,
7987+
occurredAt: now,
7988+
payload: { ...rootNode, status: "interrupted", completedAt: now },
7989+
});
7990+
}
7991+
if (providerThread !== undefined) {
7992+
yield* emitEvent({
7993+
type: "provider-thread.updated",
7994+
threadId: run.threadId,
7995+
driver: providerThread.driver,
7996+
providerInstanceId: providerThread.providerInstanceId,
7997+
occurredAt: now,
7998+
payload: { ...providerThread, status: "idle", updatedAt: now },
7999+
});
8000+
}
78718001
});
78728002

78738003
const dispatchRunInterrupt = (

0 commit comments

Comments
 (0)