Skip to content

Commit 761e3ff

Browse files
shivamhwpclaude
andcommitted
fix(server): a usage-limit auto-resume survives a server restart
#29838 made the auto-resume an agent message. A restart before the provider starts it cancels the run, and the limit worker then rebuilds the same already-receipted resume command, so the thread never resumes. Runs now record usageLimitContinuationOfRunId. When recovery cancels a never-started run while auto-resume is armed, it gives limitRecovery a fresh request id so the worker sends the resume again, and it marks the cancelled resume's prompt as queued input so the shared timeline rule hides it. The thread resumes on its own with one resume message. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent c7537d0 commit 761e3ff

4 files changed

Lines changed: 130 additions & 1 deletion

File tree

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6265,6 +6265,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
62656265
...(command.restartContinuationOfRunId === undefined
62666266
? {}
62676267
: { restartContinuationOfRunId: command.restartContinuationOfRunId }),
6268+
...(command.usageLimitContinuationOfRunId === undefined
6269+
? {}
6270+
: { usageLimitContinuationOfRunId: command.usageLimitContinuationOfRunId }),
62686271
...(continuedTaskId === undefined ? {} : { delegatedTaskId: continuedTaskId }),
62696272
...(freshContext ? { freshContext: true } : {}),
62706273
};
@@ -6645,6 +6648,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
66456648
...(command.restartContinuationOfRunId === undefined
66466649
? {}
66476650
: { restartContinuationOfRunId: command.restartContinuationOfRunId }),
6651+
...(command.usageLimitContinuationOfRunId === undefined
6652+
? {}
6653+
: { usageLimitContinuationOfRunId: command.usageLimitContinuationOfRunId }),
66486654
...(dispatchMode.type === "defer_start" && dispatchMode.workspaceStrategy !== undefined
66496655
? { workspacePreparation: dispatchMode.workspaceStrategy }
66506656
: {}),
@@ -7431,6 +7437,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
74317437
...(command.restartContinuationOfRunId === undefined
74327438
? {}
74337439
: { restartContinuationOfRunId: command.restartContinuationOfRunId }),
7440+
...(command.usageLimitContinuationOfRunId === undefined
7441+
? {}
7442+
: { usageLimitContinuationOfRunId: command.usageLimitContinuationOfRunId }),
74347443
...(continuedTaskId === undefined ? {} : { delegatedTaskId: continuedTaskId }),
74357444
...wakeWorkStartedAt(projection.runs, command),
74367445
};

‎apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -440,6 +440,64 @@ export const make = Effect.gen(function* () {
440440
},
441441
});
442442
}
443+
const limitRecovery = projection.thread.limitRecovery;
444+
// Cancelling a never-started run leaves the thread limited by the same run, so a resume sent
445+
// before the restart spent the worker's request id. A fresh id lets the worker send it again.
446+
if (limitRecovery?.autoResume === true && runs.some((run) => run.startedAt === null)) {
447+
events.push({
448+
id: yield* allocateEventId(),
449+
type: "thread.metadata-updated",
450+
threadId: projection.thread.id,
451+
occurredAt: now,
452+
payload: {
453+
...projection.thread,
454+
limitRecovery: { ...limitRecovery, requestId: commandId },
455+
updatedAt: now,
456+
},
457+
});
458+
}
459+
// The worker sends the auto-resume again, so the unsent copy leaves the transcript the way
460+
// cancelled queued input does.
461+
const unsentResumeRunIds = runs.flatMap((run) =>
462+
run.startedAt === null && run.usageLimitContinuationOfRunId !== undefined ? [run.id] : [],
463+
);
464+
const unsentResumePrompts =
465+
unsentResumeRunIds.length === 0
466+
? []
467+
: (yield* projections
468+
.getThreadRecords(projection.thread.id, ["turnItems"], {
469+
turnItemTypes: ["user_message"],
470+
turnItemRunIds: unsentResumeRunIds,
471+
})
472+
.pipe(
473+
Effect.mapError(
474+
(cause) =>
475+
new ProviderRuntimeRecoveryError({
476+
operation: "read-projections",
477+
threadId: projection.thread.id,
478+
cause,
479+
}),
480+
),
481+
)).turnItems;
482+
const unsentResumePromptIds = new Set(unsentResumePrompts.map((item) => item.id));
483+
for (const item of unsentResumePrompts) {
484+
if (item.type !== "user_message") continue;
485+
const active =
486+
item.status === "pending" || item.status === "running" || item.status === "waiting";
487+
events.push({
488+
id: yield* allocateEventId(),
489+
type: "turn-item.updated",
490+
threadId: projection.thread.id,
491+
...(item.runId === null ? {} : { runId: item.runId }),
492+
occurredAt: now,
493+
payload: {
494+
...item,
495+
...(active ? { status: "cancelled", completedAt: now } : {}),
496+
inputIntent: "queued_turn",
497+
updatedAt: now,
498+
},
499+
});
500+
}
443501
for (const question of questions) {
444502
events.push({
445503
id: yield* allocateEventId(),
@@ -550,6 +608,7 @@ export const make = Effect.gen(function* () {
550608
for (const item of projection.turnItems.filter(
551609
(candidate) =>
552610
candidate.runId === run.id &&
611+
!unsentResumePromptIds.has(candidate.id) &&
553612
(candidate.nodeId === null || !messageRequestNodeIds.has(candidate.nodeId)) &&
554613
!isAppOwnedDelegationItem(candidate) &&
555614
// A link card waits on the user, not the run, and answers by message.

‎apps/server/src/orchestration-v2/runtimeLayer.test.ts‎

Lines changed: 61 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { PullRequestProviderError } from "@t3tools/source-control-core/server/PullRequestProvider";
22
import { SourceControlProviderKind } from "@t3tools/contracts";
33
import { limitRecoveryCommand } from "./UsageLimitRecoveryWorker.ts";
4+
import { createOrchestrationV2TurnItemVisibility } from "@t3tools/shared/orchestrationV2Timeline";
45
import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderSessions";
56
import * as SourceControlProviderRegistry from "../sourceControl/SourceControlProviderRegistry.ts";
67
import * as NodeCrypto from "@effect/platform-node/NodeCrypto";
@@ -8041,6 +8042,7 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
80418042

80428043
it.effect.each([
80438044
"resume",
8045+
"resume-started",
80448046
"queued-resume",
80458047
"rearm",
80468048
"snooze-race",
@@ -8519,17 +8521,74 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
85198521
yield* orchestrator.dispatch(resume);
85208522
}
85218523
const after = yield* orchestrator.getThreadProjection(threadId);
8522-
if (scenario === "resume") {
8524+
if (scenario === "resume" || scenario === "resume-started") {
85238525
assert.deepInclude(after.messages.at(-1), {
85248526
text: "Continue where you left off.",
85258527
createdBy: "agent",
85268528
creationSource: "server",
85278529
});
8530+
const continuation = after.runs.at(-1)!;
8531+
assert.equal(continuation.status, "starting");
8532+
if (scenario === "resume-started") {
8533+
const startedAt = yield* DateTime.now;
8534+
yield* events.write({
8535+
events: [
8536+
{
8537+
id: EventId.make("recovery:continuation-started:resume-started"),
8538+
type: "run.updated",
8539+
threadId,
8540+
runId: continuation.id,
8541+
occurredAt: startedAt,
8542+
payload: { ...continuation, status: "running", startedAt },
8543+
},
8544+
],
8545+
});
8546+
}
8547+
yield* (yield* ProviderRuntimeRecoveryService.ProviderRuntimeRecoveryService).reconcile(
8548+
"startup",
8549+
);
8550+
const restartedShell = (yield* orchestrator.getShellSnapshot()).threads.find(
8551+
(thread) => thread.id === threadId,
8552+
)!;
8553+
const retry = limitRecoveryCommand(
8554+
restartedShell,
8555+
true,
8556+
DateTime.toEpochMillis(yield* DateTime.now),
8557+
);
8558+
if (scenario === "resume-started") {
8559+
assert.isNull(retry);
8560+
assert.equal(
8561+
restartedShell.limitRecovery?.requestId,
8562+
after.thread.limitRecovery?.requestId,
8563+
);
8564+
} else {
8565+
assert(retry?.type === "message.dispatch");
8566+
assert.notEqual(retry.commandId, resume!.commandId);
8567+
yield* orchestrator.dispatch(retry);
8568+
yield* orchestrator.dispatch(retry);
8569+
const resumed = yield* orchestrator.getThreadProjection(threadId);
8570+
assert.deepEqual(
8571+
resumed.runs.map((row) => row.status),
8572+
["failed", "cancelled", "starting"],
8573+
);
8574+
const visible = createOrchestrationV2TurnItemVisibility({
8575+
runs: resumed.runs,
8576+
attempts: resumed.attempts,
8577+
items: resumed.turnItems,
8578+
});
8579+
assert.deepEqual(
8580+
resumed.turnItems.flatMap((item) =>
8581+
item.type === "user_message" && visible(item) ? [item.runId] : [],
8582+
),
8583+
[run.id, resumed.runs[2]!.id],
8584+
);
8585+
}
85288586
}
85298587
assert.lengthOf(
85308588
after.runs,
85318589
before.runs.length +
85328590
(scenario === "resume" ||
8591+
scenario === "resume-started" ||
85338592
stopBeforeArm ||
85348593
scenario === "queued-resume" ||
85358594
scenario === "independent-patches" ||
@@ -8541,6 +8600,7 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
85418600
after.messages,
85428601
before.messages.length +
85438602
(scenario === "resume" ||
8603+
scenario === "resume-started" ||
85448604
stopBeforeArm ||
85458605
scenario === "queued-resume" ||
85468606
scenario === "independent-patches" ||

‎packages/contracts/src/orchestrationV2.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -692,6 +692,7 @@ export const OrchestrationV2Run = Schema.Struct({
692692
checkpointId: Schema.NullOr(CheckpointId),
693693
contextHandoffId: Schema.NullOr(ContextHandoffId),
694694
restartContinuationOfRunId: Schema.optional(RunId),
695+
usageLimitContinuationOfRunId: Schema.optional(RunId),
695696
delegatedTaskId: Schema.optional(NodeId),
696697
workStartedAt: Schema.optional(Schema.DateTimeUtc),
697698
restartCancelledBackgroundWork: Schema.optional(

0 commit comments

Comments
 (0)