Skip to content

Commit 240446e

Browse files
shivamhwpclaude
andauthored
fix(server): a usage-limit auto-resume survives a server restart (#31292)
Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent d5fc29f commit 240446e

4 files changed

Lines changed: 155 additions & 1 deletion

File tree

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6279,6 +6279,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
62796279
...(command.restartContinuationOfRunId === undefined
62806280
? {}
62816281
: { restartContinuationOfRunId: command.restartContinuationOfRunId }),
6282+
...(command.usageLimitContinuationOfRunId === undefined
6283+
? {}
6284+
: { usageLimitContinuationOfRunId: command.usageLimitContinuationOfRunId }),
62826285
...(continuedTaskId === undefined ? {} : { delegatedTaskId: continuedTaskId }),
62836286
...(freshContext ? { freshContext: true } : {}),
62846287
};
@@ -6659,6 +6662,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
66596662
...(command.restartContinuationOfRunId === undefined
66606663
? {}
66616664
: { restartContinuationOfRunId: command.restartContinuationOfRunId }),
6665+
...(command.usageLimitContinuationOfRunId === undefined
6666+
? {}
6667+
: { usageLimitContinuationOfRunId: command.usageLimitContinuationOfRunId }),
66626668
...(dispatchMode.type === "defer_start" && dispatchMode.workspaceStrategy !== undefined
66636669
? { workspacePreparation: dispatchMode.workspaceStrategy }
66646670
: {}),
@@ -7445,6 +7451,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
74457451
...(command.restartContinuationOfRunId === undefined
74467452
? {}
74477453
: { restartContinuationOfRunId: command.restartContinuationOfRunId }),
7454+
...(command.usageLimitContinuationOfRunId === undefined
7455+
? {}
7456+
: { usageLimitContinuationOfRunId: command.usageLimitContinuationOfRunId }),
74487457
...(continuedTaskId === undefined ? {} : { delegatedTaskId: continuedTaskId }),
74497458
...wakeWorkStartedAt(projection.runs, command),
74507459
};

‎apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -442,6 +442,64 @@ export const make = Effect.gen(function* () {
442442
},
443443
});
444444
}
445+
const unsentResumeRunIds = runs.flatMap((run) =>
446+
run.startedAt === null && run.usageLimitContinuationOfRunId !== undefined ? [run.id] : [],
447+
);
448+
const limitRecovery = projection.thread.limitRecovery;
449+
// The unsent resume spent the worker's request id, so a fresh one lets it send again. An
450+
// untagged resume from an older server stays cancelled: its copy couldn't be hidden.
451+
if (limitRecovery?.autoResume === true && unsentResumeRunIds.length > 0) {
452+
events.push({
453+
id: yield* allocateEventId(),
454+
type: "thread.metadata-updated",
455+
threadId: projection.thread.id,
456+
occurredAt: now,
457+
payload: {
458+
...projection.thread,
459+
limitRecovery: { ...limitRecovery, requestId: commandId },
460+
updatedAt: now,
461+
},
462+
});
463+
}
464+
// The worker sends the auto-resume again, so the unsent copy leaves the transcript the way
465+
// cancelled queued input does.
466+
const unsentResumePrompts =
467+
unsentResumeRunIds.length === 0
468+
? []
469+
: (yield* projections
470+
.getThreadRecords(projection.thread.id, ["turnItems"], {
471+
turnItemTypes: ["user_message"],
472+
turnItemRunIds: unsentResumeRunIds,
473+
})
474+
.pipe(
475+
Effect.mapError(
476+
(cause) =>
477+
new ProviderRuntimeRecoveryError({
478+
operation: "read-projections",
479+
threadId: projection.thread.id,
480+
cause,
481+
}),
482+
),
483+
)).turnItems;
484+
const unsentResumePromptIds = new Set(unsentResumePrompts.map((item) => item.id));
485+
for (const item of unsentResumePrompts) {
486+
if (item.type !== "user_message") continue;
487+
const active =
488+
item.status === "pending" || item.status === "running" || item.status === "waiting";
489+
events.push({
490+
id: yield* allocateEventId(),
491+
type: "turn-item.updated",
492+
threadId: projection.thread.id,
493+
...(item.runId === null ? {} : { runId: item.runId }),
494+
occurredAt: now,
495+
payload: {
496+
...item,
497+
...(active ? { status: "cancelled", completedAt: now } : {}),
498+
inputIntent: "queued_turn",
499+
updatedAt: now,
500+
},
501+
});
502+
}
445503
for (const question of questions) {
446504
events.push({
447505
id: yield* allocateEventId(),
@@ -552,6 +610,7 @@ export const make = Effect.gen(function* () {
552610
for (const item of projection.turnItems.filter(
553611
(candidate) =>
554612
candidate.runId === run.id &&
613+
!unsentResumePromptIds.has(candidate.id) &&
555614
(candidate.nodeId === null || !messageRequestNodeIds.has(candidate.nodeId)) &&
556615
!isAppOwnedDelegationItem(candidate) &&
557616
// A link card waits on the user, not the run, and answers by message.

‎apps/server/src/orchestration-v2/runtimeLayer.test.ts‎

Lines changed: 86 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { PullRequestProviderError } from "@t3tools/source-control-core/server/PullRequestProvider";
22
import { SourceControlProviderKind } from "@t3tools/contracts";
33
import { limitRecoveryCommand } from "./UsageLimitRecoveryWorker.ts";
4+
import { createOrchestrationV2TurnItemVisibility } from "@t3tools/shared/orchestrationV2Timeline";
45
import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderSessions";
56
import * as SourceControlProviderRegistry from "../sourceControl/SourceControlProviderRegistry.ts";
67
import * as NodeCrypto from "@effect/platform-node/NodeCrypto";
@@ -8136,6 +8137,8 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
81368137

81378138
it.effect.each([
81388139
"resume",
8140+
"resume-started",
8141+
"resume-untagged",
81398142
"queued-resume",
81408143
"rearm",
81418144
"snooze-race",
@@ -8614,17 +8617,97 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
86148617
yield* orchestrator.dispatch(resume);
86158618
}
86168619
const after = yield* orchestrator.getThreadProjection(threadId);
8617-
if (scenario === "resume") {
8620+
if (
8621+
scenario === "resume" ||
8622+
scenario === "resume-started" ||
8623+
scenario === "resume-untagged"
8624+
) {
86188625
assert.deepInclude(after.messages.at(-1), {
86198626
text: "Continue where you left off.",
86208627
createdBy: "agent",
86218628
creationSource: "server",
86228629
});
8630+
const continuation = after.runs.at(-1)!;
8631+
assert.equal(continuation.status, "starting");
8632+
if (scenario === "resume-started") {
8633+
const startedAt = yield* DateTime.now;
8634+
yield* events.write({
8635+
events: [
8636+
{
8637+
id: EventId.make("recovery:continuation-started:resume-started"),
8638+
type: "run.updated",
8639+
threadId,
8640+
runId: continuation.id,
8641+
occurredAt: startedAt,
8642+
payload: { ...continuation, status: "running", startedAt },
8643+
},
8644+
],
8645+
});
8646+
}
8647+
if (scenario === "resume-untagged") {
8648+
// A server older than the run tag stored the resume run without it.
8649+
const { usageLimitContinuationOfRunId: _, ...untagged } = continuation;
8650+
yield* events.write({
8651+
events: [
8652+
{
8653+
id: EventId.make("recovery:continuation-untagged"),
8654+
type: "run.updated",
8655+
threadId,
8656+
runId: continuation.id,
8657+
occurredAt: yield* DateTime.now,
8658+
payload: untagged,
8659+
},
8660+
],
8661+
});
8662+
}
8663+
yield* (yield* ProviderRuntimeRecoveryService.ProviderRuntimeRecoveryService).reconcile(
8664+
"startup",
8665+
);
8666+
const restartedShell = (yield* orchestrator.getShellSnapshot()).threads.find(
8667+
(thread) => thread.id === threadId,
8668+
)!;
8669+
const retry = limitRecoveryCommand(
8670+
restartedShell,
8671+
true,
8672+
DateTime.toEpochMillis(yield* DateTime.now),
8673+
);
8674+
if (scenario === "resume-started") assert.isNull(retry);
8675+
else assert(retry?.type === "message.dispatch");
8676+
if (scenario === "resume") {
8677+
assert.notEqual(retry!.commandId, resume!.commandId);
8678+
} else {
8679+
assert.equal(
8680+
restartedShell.limitRecovery?.requestId,
8681+
after.thread.limitRecovery?.requestId,
8682+
);
8683+
}
8684+
if (retry !== null) {
8685+
yield* orchestrator.dispatch(retry);
8686+
yield* orchestrator.dispatch(retry);
8687+
}
8688+
const resumed = yield* orchestrator.getThreadProjection(threadId);
8689+
const visible = createOrchestrationV2TurnItemVisibility({
8690+
runs: resumed.runs,
8691+
attempts: resumed.attempts,
8692+
items: resumed.turnItems,
8693+
});
8694+
assert.deepEqual(
8695+
resumed.turnItems.flatMap((item) =>
8696+
item.type === "user_message" && visible(item) ? [item.runId] : [],
8697+
),
8698+
[run.id, resumed.runs.at(-1)!.id],
8699+
);
8700+
assert.deepEqual(
8701+
resumed.runs.map((row) => row.status),
8702+
scenario === "resume" ? ["failed", "cancelled", "starting"] : ["failed", "cancelled"],
8703+
);
86238704
}
86248705
assert.lengthOf(
86258706
after.runs,
86268707
before.runs.length +
86278708
(scenario === "resume" ||
8709+
scenario === "resume-started" ||
8710+
scenario === "resume-untagged" ||
86288711
stopBeforeArm ||
86298712
scenario === "queued-resume" ||
86308713
scenario === "independent-patches" ||
@@ -8636,6 +8719,8 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
86368719
after.messages,
86378720
before.messages.length +
86388721
(scenario === "resume" ||
8722+
scenario === "resume-started" ||
8723+
scenario === "resume-untagged" ||
86398724
stopBeforeArm ||
86408725
scenario === "queued-resume" ||
86418726
scenario === "independent-patches" ||

‎packages/contracts/src/orchestrationV2.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -692,6 +692,7 @@ export const OrchestrationV2Run = Schema.Struct({
692692
checkpointId: Schema.NullOr(CheckpointId),
693693
contextHandoffId: Schema.NullOr(ContextHandoffId),
694694
restartContinuationOfRunId: Schema.optional(RunId),
695+
usageLimitContinuationOfRunId: Schema.optional(RunId),
695696
delegatedTaskId: Schema.optional(NodeId),
696697
workStartedAt: Schema.optional(Schema.DateTimeUtc),
697698
restartCancelledBackgroundWork: Schema.optional(

0 commit comments

Comments
 (0)