Skip to content

Commit 71ef26e

Browse files
fix(server): runs cancelled by a server restart say so in the thread
Startup and shutdown recovery cancelled running work with no item saying why, so the run looked like a provider cancel or a user Stop. Recovery now adds a completed system_notice ("Server restarted" / "Server shut down") to each run it cancels. Its id derives from the run, so replays update it in place, and EventSink slots it after the run's existing items. Queued runs that never started get no notice. Fixes #27048 Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent d10e8db commit 71ef26e

3 files changed

Lines changed: 47 additions & 4 deletions

File tree

‎apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -795,6 +795,7 @@ it.effect(
795795
["provider-turn.updated", "cancelled"],
796796
["message.updated", null],
797797
["turn-item.updated", "cancelled"],
798+
["turn-item.updated", "completed"],
798799
["provider-thread.updated", "idle"],
799800
["provider-session.updated", "stopped"],
800801
],

‎apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -489,6 +489,42 @@ export const make = Effect.gen(function* () {
489489
payload: cancelledItem(item, now),
490490
});
491491
}
492+
// A queued run never did provider work, so there is nothing to explain.
493+
if (run.status === "queued") continue;
494+
events.push({
495+
id: yield* allocateEventId(),
496+
type: "turn-item.updated",
497+
threadId: projection.thread.id,
498+
runId: run.id,
499+
...(run.rootNodeId === null ? {} : { nodeId: run.rootNodeId }),
500+
providerInstanceId: run.providerInstanceId,
501+
occurredAt: now,
502+
payload: {
503+
id: ids.derive.runSignalTurnItem({ runId: run.id, signal: "server-recovery" }),
504+
threadId: projection.thread.id,
505+
runId: run.id,
506+
nodeId: run.rootNodeId,
507+
providerThreadId: run.providerThreadId,
508+
providerTurnId: null,
509+
nativeItemRef: null,
510+
parentItemId: null,
511+
// EventSink places the item after the run's existing items.
512+
ordinal:
513+
Math.max(
514+
0,
515+
...projection.turnItems
516+
.filter((item) => item.runId === run.id)
517+
.map((item) => item.ordinal),
518+
) + 1,
519+
status: "completed",
520+
title: trigger === "startup" ? "Server restarted" : "Server shut down",
521+
startedAt: now,
522+
completedAt: now,
523+
updatedAt: now,
524+
type: "system_notice",
525+
message: detail,
526+
},
527+
});
492528
}
493529
// Blocking waits die with their turn; let child results wake the parent continuation as async tasks.
494530
const cutRunIds = new Set(runs.map((run) => run.id));

‎apps/server/src/orchestration-v2/runtimeLayer.test.ts‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1324,10 +1324,16 @@ it.layer(layerTest)("OrchestrationV2LayerLive", (it) => {
13241324
assert.equal(native.runtimeRequests[0]?.responseCapability.type, "not_resumable");
13251325
yield* recovery.reconcile("shutdown");
13261326
yield* recovery.reconcile("startup");
1327-
assert.deepEqual(
1328-
(yield* orchestrator.getThreadProjection(threadId)).runtimeRequests,
1329-
recovered.runtimeRequests,
1330-
);
1327+
const rerecovered = yield* orchestrator.getThreadProjection(threadId);
1328+
assert.deepEqual(rerecovered.runtimeRequests, recovered.runtimeRequests);
1329+
const runItems = rerecovered.turnItems.filter((item) => item.runId === runId);
1330+
const notices = runItems.filter((item) => item.type === "system_notice");
1331+
assert.lengthOf(notices, scope === "run" ? 1 : 0);
1332+
if (scope === "run") {
1333+
assert.equal(notices[0]?.title, "Server restarted");
1334+
assert.equal(notices[0]?.nodeId, recovered.runs[0]?.rootNodeId);
1335+
assert.equal(notices[0]?.ordinal, Math.max(...runItems.map((item) => item.ordinal)));
1336+
}
13311337
yield* orchestrator.dispatch({
13321338
type: "runtime-request.respond",
13331339
commandId: CommandId.make(`recovery-answer-${trigger}-${scope}`),

0 commit comments

Comments
 (0)