Skip to content

Commit d806251

Browse files
fix(server): runs cancelled by a server restart say why in the thread (#27192)
Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent 835e726 commit d806251

3 files changed

Lines changed: 47 additions & 4 deletions

File tree

‎apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -793,6 +793,7 @@ it.effect(
793793
["turn-item.updated", "cancelled"],
794794
["turn-item.updated", "cancelled"],
795795
["turn-item.updated", "cancelled"],
796+
["turn-item.updated", "completed"],
796797
["provider-thread.updated", "idle"],
797798
["provider-session.updated", "stopped"],
798799
],

‎apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -634,6 +634,36 @@ export const make = Effect.gen(function* () {
634634
payload: cancelledItem(item, now),
635635
});
636636
}
637+
// Queued runs and unsent resumes leave no visible turn behind, so there is nothing to explain.
638+
if (run.status === "queued" || unsentResumeRunIds.includes(run.id)) continue;
639+
events.push({
640+
id: yield* allocateEventId(),
641+
type: "turn-item.updated",
642+
threadId: projection.thread.id,
643+
runId: run.id,
644+
...(run.rootNodeId === null ? {} : { nodeId: run.rootNodeId }),
645+
providerInstanceId: run.providerInstanceId,
646+
occurredAt: now,
647+
payload: {
648+
id: ids.derive.runSignalTurnItem({ runId: run.id, signal: "server-recovery" }),
649+
threadId: projection.thread.id,
650+
runId: run.id,
651+
nodeId: run.rootNodeId,
652+
providerThreadId: run.providerThreadId,
653+
providerTurnId: null,
654+
nativeItemRef: null,
655+
parentItemId: null,
656+
// EventSink allocates the real position, after the run's existing items.
657+
ordinal: 0,
658+
status: "completed",
659+
title: trigger === "startup" ? "Server restarted" : "Server shut down",
660+
startedAt: now,
661+
completedAt: now,
662+
updatedAt: now,
663+
type: "system_notice",
664+
message: detail,
665+
},
666+
});
637667
}
638668
// Blocking waits die with their turn; let child results wake the parent continuation as async tasks.
639669
const cutRunIds = new Set(runs.map((run) => run.id));

‎apps/server/src/orchestration-v2/runtimeLayer.test.ts‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1591,10 +1591,16 @@ it.layer(layerTest)("OrchestrationV2LayerLive", (it) => {
15911591
assert.equal(native.runtimeRequests[0]?.responseCapability.type, "not_resumable");
15921592
yield* recovery.reconcile("shutdown");
15931593
yield* recovery.reconcile("startup");
1594-
assert.deepEqual(
1595-
(yield* orchestrator.getThreadProjection(threadId)).runtimeRequests,
1596-
recovered.runtimeRequests,
1597-
);
1594+
const rerecovered = yield* orchestrator.getThreadProjection(threadId);
1595+
assert.deepEqual(rerecovered.runtimeRequests, recovered.runtimeRequests);
1596+
const runItems = rerecovered.turnItems.filter((item) => item.runId === runId);
1597+
const notices = runItems.filter((item) => item.type === "system_notice");
1598+
assert.lengthOf(notices, scope === "run" ? 1 : 0);
1599+
if (scope === "run") {
1600+
assert.equal(notices[0]?.title, "Server restarted");
1601+
assert.equal(notices[0]?.nodeId, recovered.runs[0]?.rootNodeId);
1602+
assert.equal(notices[0]?.ordinal, Math.max(...runItems.map((item) => item.ordinal)));
1603+
}
15981604
yield* orchestrator.dispatch({
15991605
type: "runtime-request.respond",
16001606
commandId: CommandId.make(`recovery-answer-${trigger}-${scope}`),
@@ -9416,6 +9422,12 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
94169422
),
94179423
[run.id, resumed.runs.at(-1)!.id],
94189424
);
9425+
assert.deepEqual(
9426+
resumed.turnItems.flatMap((item) =>
9427+
item.type === "system_notice" && item.title === "Server restarted" ? [item.runId] : [],
9428+
),
9429+
scenario === "resume" ? [] : [continuation.id],
9430+
);
94199431
assert.deepEqual(
94209432
resumed.runs.map((row) => row.status),
94219433
scenario === "resume" ? ["failed", "cancelled", "starting"] : ["failed", "cancelled"],

0 commit comments

Comments
 (0)