Skip to content

Commit e13cf9e

Browse files
committed
fix(server): Stop blocks pending restart continuations
1 parent 658ee49 commit e13cf9e

2 files changed

Lines changed: 164 additions & 52 deletions

File tree

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 67 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -8009,14 +8009,70 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
80098009
mapDispatchError(command),
80108010
);
80118011

8012+
/** Records Stop for a run without replacing a marker already written by its interrupt. */
8013+
const markStoppedRun = Effect.fnUntraced(function* (input: {
8014+
readonly command: Extract<
8015+
OrchestrationV2ServerCommand,
8016+
{ readonly type: "run.interrupt" | "thread.stop" }
8017+
>;
8018+
readonly events: Ref.Ref<Array<OrchestrationV2DomainEvent>>;
8019+
readonly thread: OrchestrationV2AppThread;
8020+
readonly run: OrchestrationV2Run;
8021+
readonly now: DateTime.Utc;
8022+
}) {
8023+
const { command, events, thread, run, now } = input;
8024+
if (run.rootNodeId === null || run.providerThreadId === null) return;
8025+
const pendingItems = (yield* Ref.get(events)).flatMap((event) =>
8026+
event.type === "turn-item.updated" ? [event.payload] : [],
8027+
);
8028+
if (
8029+
pendingItems.some((item) => item.runId === run.id && item.type === "run_interrupt_request") ||
8030+
(yield* stopReachedRun(command, thread.id, run.id))
8031+
) {
8032+
return;
8033+
}
8034+
yield* emit(
8035+
events,
8036+
command,
8037+
)({
8038+
type: "turn-item.updated",
8039+
threadId: thread.id,
8040+
runId: run.id,
8041+
nodeId: run.rootNodeId,
8042+
providerInstanceId: run.providerInstanceId,
8043+
occurredAt: now,
8044+
payload: {
8045+
id: idAllocator.derive.runSignalTurnItem({ runId: run.id, signal: "interrupt-request" }),
8046+
threadId: thread.id,
8047+
runId: run.id,
8048+
nodeId: run.rootNodeId,
8049+
providerThreadId: run.providerThreadId,
8050+
providerTurnId: null,
8051+
nativeItemRef: null,
8052+
parentItemId: null,
8053+
ordinal: yield* nextTurnItemOrdinal({ thread, turnItems: pendingItems }),
8054+
status: "completed",
8055+
title: "Interrupt requested",
8056+
startedAt: now,
8057+
completedAt: now,
8058+
updatedAt: now,
8059+
type: "run_interrupt_request",
8060+
message: command.reason ?? "Interrupt requested",
8061+
},
8062+
});
8063+
});
8064+
80128065
/**
80138066
* What Stop holds besides the run it interrupts: queued runs wait for the user, the
80148067
* thread's pull request watches end, and every wake its delegated tasks still owe is
80158068
* dropped, since Stop stops those tasks too. Nothing automatic starts the thread again.
80168069
* `cohortRunIds` are dropped even when they owe nothing, like a plain interrupt's.
80178070
*/
80188071
const holdStoppedThread = (input: {
8019-
readonly command: OrchestrationV2ServerCommand;
8072+
readonly command: Extract<
8073+
OrchestrationV2ServerCommand,
8074+
{ readonly type: "run.interrupt" | "thread.stop" }
8075+
>;
80208076
readonly events: Ref.Ref<Array<OrchestrationV2DomainEvent>>;
80218077
readonly projection: Pick<OrchestrationV2ThreadProjection, "thread" | "runs" | "subagents">;
80228078
readonly cohortRunIds: ReadonlyArray<RunId>;
@@ -8026,6 +8082,14 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
80268082
const emitEvent = emit(input.events, input.command);
80278083
const { thread, runs, subagents } = input.projection;
80288084
for (const run of runs) {
8085+
// A resumed queue can have a lower ordinal than a pending continuation source.
8086+
// Stop must reach those sources even when interrupting the resumed run succeeds.
8087+
if (
8088+
run.status === "cancelled" &&
8089+
(yield* awaitsRestartContinuation(run).pipe(mapDispatchError(input.command)))
8090+
) {
8091+
yield* markStoppedRun({ ...input, thread, run });
8092+
}
80298093
if (run.status !== "queued" || run.queueHeld === true) continue;
80308094
yield* emitEvent({
80318095
type: "run.updated",
@@ -8592,63 +8656,14 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
85928656
}
85938657
const now = yield* DateTime.now;
85948658
// Record that Stop reached the turn it could not interrupt and the thread's last executed
8595-
// run, so a late tool call from their agent starts nothing, and every run a restart cut,
8596-
// so its continuation never starts.
8659+
// run, so a late tool call from their agent starts nothing.
85978660
const executed = latestExecutedRun(projection.runs);
85988661
const reached = new Map<RunId, OrchestrationV2Run>();
85998662
if (target !== undefined) reached.set(target.id, target);
86008663
if (executed !== null && !hasLiveRun({ runs: [executed] }))
86018664
reached.set(executed.id, executed);
8602-
for (const run of projection.runs) {
8603-
if (
8604-
run.status === "cancelled" &&
8605-
(executed === null || run.ordinal >= executed.ordinal) &&
8606-
(yield* awaitsRestartContinuation(run).pipe(mapDispatchError(command)))
8607-
) {
8608-
reached.set(run.id, run);
8609-
}
8610-
}
8611-
let ordinal = yield* nextTurnItemOrdinal(projection);
86128665
for (const run of reached.values()) {
8613-
if (
8614-
run.rootNodeId === null ||
8615-
run.providerThreadId === null ||
8616-
(yield* stopReachedRun(command, command.threadId, run.id))
8617-
) {
8618-
continue;
8619-
}
8620-
yield* emit(
8621-
events,
8622-
command,
8623-
)({
8624-
type: "turn-item.updated",
8625-
threadId: command.threadId,
8626-
runId: run.id,
8627-
nodeId: run.rootNodeId,
8628-
providerInstanceId: run.providerInstanceId,
8629-
occurredAt: now,
8630-
payload: {
8631-
id: idAllocator.derive.runSignalTurnItem({
8632-
runId: run.id,
8633-
signal: "interrupt-request",
8634-
}),
8635-
threadId: command.threadId,
8636-
runId: run.id,
8637-
nodeId: run.rootNodeId,
8638-
providerThreadId: run.providerThreadId,
8639-
providerTurnId: null,
8640-
nativeItemRef: null,
8641-
parentItemId: null,
8642-
ordinal: ordinal++,
8643-
status: "completed",
8644-
title: "Interrupt requested",
8645-
startedAt: now,
8646-
completedAt: now,
8647-
updatedAt: now,
8648-
type: "run_interrupt_request",
8649-
message: command.reason ?? "Interrupt requested",
8650-
},
8651-
});
8666+
yield* markStoppedRun({ command, events, thread: projection.thread, run, now });
86528667
}
86538668
yield* holdStoppedThread({ command, events, projection, cohortRunIds: [], now });
86548669
return undefined;

‎apps/server/src/orchestration-v2/ThreadStop.test.ts‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import * as Exit from "effect/Exit";
1717
import * as Layer from "effect/Layer";
1818
import * as Schema from "effect/Schema";
1919
import * as SqlClient from "effect/unstable/sql/SqlClient";
20+
import * as TestClock from "effect/testing/TestClock";
2021

2122
import { SqlitePersistenceMemory } from "../persistence/Layers/Sqlite.ts";
2223
import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts";
@@ -395,6 +396,102 @@ it.effect("thread.stop keeps a restart continuation of the stopped run from star
395396
}).pipe(Effect.provide(testLayer)),
396397
);
397398

399+
it.effect.each(["thread.stop", "run.interrupt"] as const)(
400+
"a successful %s blocks a newer restart continuation when completion timestamps tie",
401+
(stopType) =>
402+
Effect.gen(function* () {
403+
const orchestrator = yield* Orchestrator.OrchestratorV2;
404+
const projections = yield* ProjectionStore.ProjectionStoreV2;
405+
const sql = yield* SqlClient.SqlClient;
406+
const threadId = ThreadId.make("thread:stop-resumed-queue");
407+
yield* createWatchingThread(threadId, 12);
408+
yield* send(threadId, "first", "start_immediately");
409+
yield* send(threadId, "queued", "queue_after_active");
410+
const now = yield* DateTime.now;
411+
const [first, queued] = (yield* orchestrator.getThreadProjection(threadId)).runs;
412+
assert.isDefined(first);
413+
assert.isDefined(queued);
414+
for (const run of [first!, queued!]) {
415+
yield* projections.apply({
416+
id: EventId.make(`event:stop-resumed-queue:hold:${run.id}`),
417+
type: "run.updated",
418+
threadId,
419+
runId: run.id,
420+
occurredAt: now,
421+
payload:
422+
run.id === first!.id
423+
? { ...run, status: "completed", completedAt: now }
424+
: { ...run, queueHeld: true },
425+
});
426+
}
427+
// A later turn runs ahead of the held queue, then recovery cancels it.
428+
yield* send(threadId, "restart source", "start_immediately");
429+
const source = (yield* orchestrator.getThreadProjection(threadId)).runs.at(-1)!;
430+
yield* projections.apply({
431+
id: EventId.make("event:stop-resumed-queue:restart-cancelled"),
432+
type: "run.updated",
433+
threadId,
434+
runId: source.id,
435+
occurredAt: now,
436+
payload: { ...source, status: "cancelled", startedAt: now, completedAt: now },
437+
});
438+
const at = DateTime.formatIso(now);
439+
yield* sql`
440+
INSERT INTO orchestration_v2_effect_outbox (
441+
effect_id, command_id, thread_id, effect_type, payload_json, status,
442+
attempt_count, available_at, created_at, updated_at
443+
) VALUES (
444+
${`effect:restart-continuation:${source.id}`}, 'command:restart', ${threadId},
445+
'provider-runtime.continue',
446+
${encodeEffectRequest({ type: "provider-runtime.continue", sourceRunId: source.id })},
447+
'pending', 0, ${at}, ${at}, ${at}
448+
)
449+
`;
450+
yield* orchestrator.dispatch({
451+
type: "queue.resume",
452+
commandId: CommandId.make("resume-older-queue"),
453+
threadId,
454+
});
455+
assert.equal(
456+
(yield* orchestrator.getThreadProjection(threadId)).runs.find(
457+
(run) => run.id === queued!.id,
458+
)?.status,
459+
"starting",
460+
);
461+
yield* orchestrator.dispatch(
462+
stopType === "thread.stop"
463+
? { type: "thread.stop", commandId: CommandId.make("stop-resumed-queue"), threadId }
464+
: {
465+
type: "run.interrupt",
466+
commandId: CommandId.make("stop-resumed-queue"),
467+
threadId,
468+
runId: queued!.id,
469+
holdQueue: true,
470+
},
471+
);
472+
const stopped = yield* orchestrator.getThreadProjection(threadId);
473+
const interrupted = stopped.runs.find((run) => run.id === queued!.id)!;
474+
assert.equal(interrupted.status, "interrupted");
475+
assert.equal(DateTime.toEpochMillis(interrupted.completedAt!), DateTime.toEpochMillis(now));
476+
yield* orchestrator.dispatch({
477+
type: "message.dispatch",
478+
commandId: CommandId.make("restart-after-stop"),
479+
threadId,
480+
messageId: MessageId.make("message:restart-after-stop"),
481+
text: "Continue.",
482+
attachments: [],
483+
dispatchMode: { type: "start_immediately" },
484+
createdBy: "agent",
485+
creationSource: "server",
486+
restartContinuationOfRunId: source.id,
487+
});
488+
assert.deepEqual(yield* threadState(threadId), {
489+
runs: ["completed", "interrupted", "cancelled"],
490+
watched: [],
491+
});
492+
}).pipe(Effect.provide(testLayer.pipe(Layer.provideMerge(TestClock.layer())))),
493+
);
494+
398495
it.effect("a delegated task that cannot be stopped fails the walk after its siblings stop", () =>
399496
Effect.gen(function* () {
400497
const orchestrator = yield* Orchestrator.OrchestratorV2;

0 commit comments

Comments
 (0)