Skip to content

Commit cc4df41

Browse files
zainmitchellclaude
andcommitted
fix(server): a queue cancelled before it ran no longer steers Stop
Stop took the newest non-queued run as the run it reaches. A queued run cancelled before it started is non-queued, so after one: - a delegated Stop retry checked its receipt bound against that cancelled run and skipped the original run, active or finished; - Stop on settled background work marked the cancelled run, leaving the run whose agent owns the work free to start a late watch. Stop and the background-work check in run.interrupt now skip a run that was cancelled before it started. The run.created bound is unchanged, so a run queued before the original Stop still belongs to it. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 92b1514 commit cc4df41

2 files changed

Lines changed: 202 additions & 2 deletions

File tree

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -566,6 +566,15 @@ function hasLiveRun(projection: Pick<OrchestrationV2ThreadProjection, "runs">):
566566
);
567567
}
568568

569+
/** A queued run cancelled before it started owns no work, so Stop never reaches it. */
570+
function latestDequeuedRun(
571+
runs: ReadonlyArray<OrchestrationV2Run>,
572+
): OrchestrationV2Run | undefined {
573+
return runs.findLast(
574+
(run) => run.status !== "queued" && !(run.status === "cancelled" && run.startedAt === null),
575+
);
576+
}
577+
569578
function withPullRequestWatch(
570579
link: ThreadPullRequestLink,
571580
watch: ThreadPullRequestWatch | undefined,
@@ -9511,7 +9520,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
95119520
? undefined
95129521
: projection.providerThreads.find((candidate) => candidate.id === run?.providerThreadId);
95139522
const hasBackgroundWork =
9514-
run?.id === projection.runs.findLast((candidate) => candidate.status !== "queued")?.id &&
9523+
run?.id === latestDequeuedRun(projection.runs)?.id &&
95159524
derivePendingBackgroundWork({
95169525
latestRun: run,
95179526
providerThreads: projection.providerThreads,
@@ -9968,7 +9977,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
99689977
: new OrchestratorProjectionError({ threadId: command.threadId, cause }),
99699978
),
99709979
);
9971-
const latestRun = projection.runs.findLast((run) => run.status !== "queued");
9980+
const latestRun = latestDequeuedRun(projection.runs);
99729981
if (command.throughSequence !== undefined && latestRun !== undefined) {
99739982
// A queued run already belonged to Stop even if it started before the delegated walk.
99749983
const existed = yield* eventSink

‎apps/server/src/orchestration-v2/ThreadStop.test.ts‎

Lines changed: 191 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1161,6 +1161,197 @@ it.effect("a partial delegated Stop retries only the original children that fail
11611161
}).pipe(Effect.provide(layerTest)),
11621162
);
11631163

1164+
it.effect.each([
1165+
{
1166+
name: "an active run past a queue cancelled since",
1167+
afterStop: ["queue", "cancel"],
1168+
expected: { runs: ["interrupted", "cancelled"], watched: [] },
1169+
},
1170+
{
1171+
name: "a finished run past a queue cancelled since",
1172+
afterStop: ["queue", "cancel", "complete"],
1173+
expected: { runs: ["completed", "cancelled"], watched: [] },
1174+
},
1175+
{
1176+
name: "a run queued before Stop that started since",
1177+
beforeStop: ["queue"],
1178+
afterStop: ["complete", "resume"],
1179+
expected: { runs: ["completed", "interrupted"], watched: [] },
1180+
},
1181+
{
1182+
name: "a run queued before Stop that started after newer work finished",
1183+
beforeStop: ["queue"],
1184+
afterStop: ["complete", "start", "complete", "resume"],
1185+
expected: { runs: ["completed", "completed", "interrupted"], watched: [] },
1186+
},
1187+
{
1188+
// Partial retries keep queue/watch changes thread-wide while the original run is current.
1189+
name: "the original run and the queue and watch added since",
1190+
afterStop: ["queue", "watch"],
1191+
expected: { runs: ["interrupted", "queued:held"], watched: [] },
1192+
},
1193+
{
1194+
name: "nothing when only newer work started",
1195+
afterStop: ["complete", "start"],
1196+
expected: { runs: ["completed", "starting"], watched: [40] },
1197+
},
1198+
{
1199+
name: "nothing when newer work finished past a queue cancelled since",
1200+
afterStop: ["complete", "start", "queue", "cancel", "complete"],
1201+
expected: { runs: ["completed", "completed", "cancelled"], watched: [40] },
1202+
},
1203+
] as const)("a Stop retry reaches $name", ({ beforeStop = [], afterStop, expected }) =>
1204+
Effect.gen(function* () {
1205+
const orchestrator = yield* Orchestrator.OrchestratorV2;
1206+
const projections = yield* ProjectionStore.ProjectionStoreV2;
1207+
const threadId = ThreadId.make(
1208+
`thread:stop-retry-bound:${afterStop.join(",")}:${beforeStop.join(",")}`,
1209+
);
1210+
let step = 0;
1211+
const apply = (action: (typeof beforeStop)[number] | (typeof afterStop)[number]) =>
1212+
Effect.gen(function* () {
1213+
step++;
1214+
if (action === "queue")
1215+
return yield* send(threadId, `queued ${step}`, "queue_after_active");
1216+
if (action === "start") return yield* send(threadId, `new ${step}`, "start_immediately");
1217+
if (action === "watch") return yield* watch(threadId, 41);
1218+
const runs = (yield* orchestrator.getThreadProjection(threadId)).runs;
1219+
if (action === "resume")
1220+
return yield* orchestrator.dispatch({
1221+
type: "queue.resume",
1222+
commandId: CommandId.make(`resume:${step}`),
1223+
threadId,
1224+
});
1225+
if (action === "cancel")
1226+
return yield* orchestrator.dispatch({
1227+
type: "queued-run.cancel",
1228+
commandId: CommandId.make(`cancel:${step}`),
1229+
threadId,
1230+
runId: runs.findLast((run) => run.status === "queued")!.id,
1231+
});
1232+
const now = yield* DateTime.now;
1233+
const run = runs.findLast((candidate) => candidate.status === "starting")!;
1234+
yield* projections.apply({
1235+
id: EventId.make(`event:stop-retry-bound:${step}`),
1236+
type: "run.updated",
1237+
threadId,
1238+
runId: run.id,
1239+
occurredAt: now,
1240+
payload: { ...run, status: "completed", startedAt: now, completedAt: now },
1241+
});
1242+
});
1243+
yield* createWatchingThread(threadId, 40);
1244+
yield* send(threadId, "original", "start_immediately");
1245+
for (const action of beforeStop) yield* apply(action);
1246+
// The original Stop's receipt bound; its walk failed before reaching this thread.
1247+
const throughSequence = yield* orchestrator.getThreadEventSequence(threadId);
1248+
for (const action of afterStop) yield* apply(action);
1249+
yield* orchestrator.dispatch({
1250+
type: "thread.stop",
1251+
commandId: CommandId.make("retry-stop"),
1252+
threadId,
1253+
throughSequence,
1254+
});
1255+
assert.deepEqual(yield* threadState(threadId), {
1256+
runs: [...expected.runs],
1257+
watched: [...expected.watched],
1258+
});
1259+
}).pipe(Effect.provide(layerTest)),
1260+
);
1261+
1262+
it.effect("thread.stop interrupts background work past a queue cancelled before it ran", () =>
1263+
Effect.gen(function* () {
1264+
const orchestrator = yield* Orchestrator.OrchestratorV2;
1265+
const projections = yield* ProjectionStore.ProjectionStoreV2;
1266+
const threadId = ThreadId.make("thread:stop-background-cancelled-queue");
1267+
yield* createWatchingThread(threadId, 42);
1268+
yield* send(threadId, "work", "start_immediately");
1269+
yield* send(threadId, "queued", "queue_after_active");
1270+
const [run, queued] = (yield* orchestrator.getThreadProjection(threadId)).runs;
1271+
yield* orchestrator.dispatch({
1272+
type: "queued-run.cancel",
1273+
commandId: CommandId.make("cancel-before-background"),
1274+
threadId,
1275+
runId: queued!.id,
1276+
});
1277+
const now = yield* DateTime.now;
1278+
const providerTurnId = ProviderTurnId.make("provider-turn:stop-background");
1279+
yield* projections.apply({
1280+
id: EventId.make("event:stop-background:turn"),
1281+
type: "provider-turn.updated",
1282+
threadId,
1283+
occurredAt: now,
1284+
payload: {
1285+
id: providerTurnId,
1286+
providerThreadId: run!.providerThreadId!,
1287+
nodeId: run!.rootNodeId!,
1288+
runAttemptId: run!.activeAttemptId!,
1289+
nativeTurnRef: null,
1290+
ordinal: 1,
1291+
status: "completed",
1292+
startedAt: now,
1293+
completedAt: now,
1294+
},
1295+
});
1296+
const backgroundId = TurnItemId.make("item:stop-background");
1297+
yield* projections.apply({
1298+
id: EventId.make("event:stop-background:item"),
1299+
type: "turn-item.updated",
1300+
threadId,
1301+
runId: run!.id,
1302+
occurredAt: now,
1303+
payload: {
1304+
id: backgroundId,
1305+
threadId,
1306+
runId: run!.id,
1307+
nodeId: run!.rootNodeId!,
1308+
providerThreadId: run!.providerThreadId!,
1309+
providerTurnId,
1310+
nativeItemRef: null,
1311+
parentItemId: null,
1312+
ordinal: 1,
1313+
status: "running",
1314+
title: "Background command",
1315+
startedAt: now,
1316+
completedAt: null,
1317+
updatedAt: now,
1318+
type: "command_execution",
1319+
input: "test background work",
1320+
},
1321+
});
1322+
yield* projections.apply({
1323+
id: EventId.make("event:stop-background:completed"),
1324+
type: "run.updated",
1325+
threadId,
1326+
runId: run!.id,
1327+
occurredAt: now,
1328+
payload: { ...run!, status: "completed", startedAt: now, completedAt: now },
1329+
});
1330+
1331+
yield* orchestrator.dispatch({
1332+
type: "thread.stop",
1333+
commandId: CommandId.make("stop-background"),
1334+
threadId,
1335+
});
1336+
const stopped = yield* orchestrator.getThreadProjection(threadId);
1337+
assert.notInclude(
1338+
["pending", "running", "waiting"],
1339+
stopped.turnItems.find((item) => item.id === backgroundId)?.status,
1340+
);
1341+
assert.deepEqual(
1342+
stopped.turnItems
1343+
.filter((item) => item.type === "run_interrupt_request")
1344+
.map((item) => item.runId),
1345+
[run!.id],
1346+
);
1347+
assert.isTrue(Exit.isFailure(yield* Effect.exit(watch(threadId, 43))));
1348+
assert.deepEqual(yield* threadState(threadId), {
1349+
runs: ["completed", "cancelled"],
1350+
watched: [],
1351+
});
1352+
}).pipe(Effect.provide(layerTest)),
1353+
);
1354+
11641355
it.effect("thread.stop on a finished thread refuses a late agent watch", () =>
11651356
Effect.gen(function* () {
11661357
const orchestrator = yield* Orchestrator.OrchestratorV2;

0 commit comments

Comments
 (0)