Skip to content

Commit 5ef11ff

Browse files
shivamhwpclaude
andcommitted
fix(server): only re-send a resume restart recovery can hide
Restart recovery now re-arms the usage-limit auto-resume only when the cancelled run carries the resume tag. An untagged resume from an older server stays cancelled, as on main, instead of showing twice, and other cancelled runs no longer trigger an extra resume. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 7fb9da4 commit 5ef11ff

2 files changed

Lines changed: 53 additions & 28 deletions

File tree

‎apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -442,10 +442,13 @@ export const make = Effect.gen(function* () {
442442
},
443443
});
444444
}
445+
const unsentResumeRunIds = runs.flatMap((run) =>
446+
run.startedAt === null && run.usageLimitContinuationOfRunId !== undefined ? [run.id] : [],
447+
);
445448
const limitRecovery = projection.thread.limitRecovery;
446-
// Cancelling a never-started run leaves the thread limited by the same run, so a resume sent
447-
// before the restart spent the worker's request id. A fresh id lets the worker send it again.
448-
if (limitRecovery?.autoResume === true && runs.some((run) => run.startedAt === null)) {
449+
// The unsent resume spent the worker's request id, so a fresh one lets it send again. An
450+
// untagged resume from an older server stays cancelled: its copy couldn't be hidden.
451+
if (limitRecovery?.autoResume === true && unsentResumeRunIds.length > 0) {
449452
events.push({
450453
id: yield* allocateEventId(),
451454
type: "thread.metadata-updated",
@@ -460,9 +463,6 @@ export const make = Effect.gen(function* () {
460463
}
461464
// The worker sends the auto-resume again, so the unsent copy leaves the transcript the way
462465
// cancelled queued input does.
463-
const unsentResumeRunIds = runs.flatMap((run) =>
464-
run.startedAt === null && run.usageLimitContinuationOfRunId !== undefined ? [run.id] : [],
465-
);
466466
const unsentResumePrompts =
467467
unsentResumeRunIds.length === 0
468468
? []

‎apps/server/src/orchestration-v2/runtimeLayer.test.ts‎

Lines changed: 47 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -8138,6 +8138,7 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
81388138
it.effect.each([
81398139
"resume",
81408140
"resume-started",
8141+
"resume-untagged",
81418142
"queued-resume",
81428143
"rearm",
81438144
"snooze-race",
@@ -8616,7 +8617,11 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
86168617
yield* orchestrator.dispatch(resume);
86178618
}
86188619
const after = yield* orchestrator.getThreadProjection(threadId);
8619-
if (scenario === "resume" || scenario === "resume-started") {
8620+
if (
8621+
scenario === "resume" ||
8622+
scenario === "resume-started" ||
8623+
scenario === "resume-untagged"
8624+
) {
86208625
assert.deepInclude(after.messages.at(-1), {
86218626
text: "Continue where you left off.",
86228627
createdBy: "agent",
@@ -8639,6 +8644,22 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
86398644
],
86408645
});
86418646
}
8647+
if (scenario === "resume-untagged") {
8648+
// A server older than the run tag stored the resume run without it.
8649+
const { usageLimitContinuationOfRunId: _, ...untagged } = continuation;
8650+
yield* events.write({
8651+
events: [
8652+
{
8653+
id: EventId.make("recovery:continuation-untagged"),
8654+
type: "run.updated",
8655+
threadId,
8656+
runId: continuation.id,
8657+
occurredAt: yield* DateTime.now,
8658+
payload: untagged,
8659+
},
8660+
],
8661+
});
8662+
}
86428663
yield* (yield* ProviderRuntimeRecoveryService.ProviderRuntimeRecoveryService).reconcile(
86438664
"startup",
86448665
);
@@ -8650,40 +8671,43 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
86508671
true,
86518672
DateTime.toEpochMillis(yield* DateTime.now),
86528673
);
8653-
if (scenario === "resume-started") {
8654-
assert.isNull(retry);
8674+
if (scenario === "resume-started") assert.isNull(retry);
8675+
else assert(retry?.type === "message.dispatch");
8676+
if (scenario === "resume") {
8677+
assert.notEqual(retry!.commandId, resume!.commandId);
8678+
} else {
86558679
assert.equal(
86568680
restartedShell.limitRecovery?.requestId,
86578681
after.thread.limitRecovery?.requestId,
86588682
);
8659-
} else {
8660-
assert(retry?.type === "message.dispatch");
8661-
assert.notEqual(retry.commandId, resume!.commandId);
8683+
}
8684+
if (retry !== null) {
86628685
yield* orchestrator.dispatch(retry);
86638686
yield* orchestrator.dispatch(retry);
8664-
const resumed = yield* orchestrator.getThreadProjection(threadId);
8665-
assert.deepEqual(
8666-
resumed.runs.map((row) => row.status),
8667-
["failed", "cancelled", "starting"],
8668-
);
8669-
const visible = createOrchestrationV2TurnItemVisibility({
8670-
runs: resumed.runs,
8671-
attempts: resumed.attempts,
8672-
items: resumed.turnItems,
8673-
});
8674-
assert.deepEqual(
8675-
resumed.turnItems.flatMap((item) =>
8676-
item.type === "user_message" && visible(item) ? [item.runId] : [],
8677-
),
8678-
[run.id, resumed.runs[2]!.id],
8679-
);
86808687
}
8688+
const resumed = yield* orchestrator.getThreadProjection(threadId);
8689+
const visible = createOrchestrationV2TurnItemVisibility({
8690+
runs: resumed.runs,
8691+
attempts: resumed.attempts,
8692+
items: resumed.turnItems,
8693+
});
8694+
assert.deepEqual(
8695+
resumed.turnItems.flatMap((item) =>
8696+
item.type === "user_message" && visible(item) ? [item.runId] : [],
8697+
),
8698+
[run.id, resumed.runs.at(-1)!.id],
8699+
);
8700+
assert.deepEqual(
8701+
resumed.runs.map((row) => row.status),
8702+
scenario === "resume" ? ["failed", "cancelled", "starting"] : ["failed", "cancelled"],
8703+
);
86818704
}
86828705
assert.lengthOf(
86838706
after.runs,
86848707
before.runs.length +
86858708
(scenario === "resume" ||
86868709
scenario === "resume-started" ||
8710+
scenario === "resume-untagged" ||
86878711
stopBeforeArm ||
86888712
scenario === "queued-resume" ||
86898713
scenario === "independent-patches" ||
@@ -8696,6 +8720,7 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
86968720
before.messages.length +
86978721
(scenario === "resume" ||
86988722
scenario === "resume-started" ||
8723+
scenario === "resume-untagged" ||
86998724
stopBeforeArm ||
87008725
scenario === "queued-resume" ||
87018726
scenario === "independent-patches" ||

0 commit comments

Comments
 (0)