Skip to content

Commit a43f47e

Browse files
shivamhwpclaude
andcommitted
fix(server): re-arm auto-resume without a run marker
Recovery re-arms whenever it cancels a never-started run while auto-resume is armed. That drops the contract field and also covers a resume sent by an older server. A resume that already started is not re-armed. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 46fb7bf commit a43f47e

4 files changed

Lines changed: 51 additions & 37 deletions

File tree

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -6155,9 +6155,6 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
61556155
...(command.restartContinuationOfRunId === undefined
61566156
? {}
61576157
: { restartContinuationOfRunId: command.restartContinuationOfRunId }),
6158-
...(command.usageLimitContinuationOfRunId === undefined
6159-
? {}
6160-
: { usageLimitContinuationOfRunId: command.usageLimitContinuationOfRunId }),
61616158
...(continuedTaskId === undefined ? {} : { delegatedTaskId: continuedTaskId }),
61626159
};
61636160
const attempt: OrchestrationV2RunAttempt = {
@@ -6520,9 +6517,6 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
65206517
...(command.restartContinuationOfRunId === undefined
65216518
? {}
65226519
: { restartContinuationOfRunId: command.restartContinuationOfRunId }),
6523-
...(command.usageLimitContinuationOfRunId === undefined
6524-
? {}
6525-
: { usageLimitContinuationOfRunId: command.usageLimitContinuationOfRunId }),
65266520
...(dispatchMode.type === "defer_start" && dispatchMode.workspaceStrategy !== undefined
65276521
? { workspacePreparation: dispatchMode.workspaceStrategy }
65286522
: {}),
@@ -7278,9 +7272,6 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
72787272
...(command.restartContinuationOfRunId === undefined
72797273
? {}
72807274
: { restartContinuationOfRunId: command.restartContinuationOfRunId }),
7281-
...(command.usageLimitContinuationOfRunId === undefined
7282-
? {}
7283-
: { usageLimitContinuationOfRunId: command.usageLimitContinuationOfRunId }),
72847275
...(continuedTaskId === undefined ? {} : { delegatedTaskId: continuedTaskId }),
72857276
...wakeWorkStartedAt(projection.runs, command),
72867277
};

‎apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts‎

Lines changed: 16 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -337,24 +337,6 @@ export const make = Effect.gen(function* () {
337337
projection.nodes.some((node) => node.id === request.nodeId && node.runId === run.id),
338338
),
339339
);
340-
const limitRecovery = projection.thread.limitRecovery;
341-
// The cancelled auto-resume spent its request id; a fresh one lets the limit worker resend it.
342-
if (
343-
limitRecovery?.autoResume === true &&
344-
unstartedRuns.some((run) => run.usageLimitContinuationOfRunId === limitRecovery.runId)
345-
) {
346-
events.push({
347-
id: yield* allocateEventId(),
348-
type: "thread.metadata-updated",
349-
threadId: projection.thread.id,
350-
occurredAt: now,
351-
payload: {
352-
...projection.thread,
353-
limitRecovery: { ...limitRecovery, requestId: commandId },
354-
updatedAt: now,
355-
},
356-
});
357-
}
358340
const queuedRuns = projection.runs.filter((run) => run.status === "queued");
359341
const pendingRuns = [...unstartedRuns, ...queuedRuns];
360342
const queuedMessages =
@@ -458,6 +440,22 @@ export const make = Effect.gen(function* () {
458440
},
459441
});
460442
}
443+
const limitRecovery = projection.thread.limitRecovery;
444+
// Cancelling a never-started run leaves the thread limited by the same run, so a resume sent
445+
// before the restart spent the worker's request id. A fresh id lets the worker send it again.
446+
if (limitRecovery?.autoResume === true && runs.some((run) => run.startedAt === null)) {
447+
events.push({
448+
id: yield* allocateEventId(),
449+
type: "thread.metadata-updated",
450+
threadId: projection.thread.id,
451+
occurredAt: now,
452+
payload: {
453+
...projection.thread,
454+
limitRecovery: { ...limitRecovery, requestId: commandId },
455+
updatedAt: now,
456+
},
457+
});
458+
}
461459
for (const question of questions) {
462460
events.push({
463461
id: yield* allocateEventId(),

‎apps/server/src/orchestration-v2/runtimeLayer.test.ts‎

Lines changed: 35 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -7993,6 +7993,7 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
79937993

79947994
it.effect.each([
79957995
"resume",
7996+
"resume-started",
79967997
"queued-resume",
79977998
"rearm",
79987999
"snooze-race",
@@ -8471,14 +8472,29 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
84718472
yield* orchestrator.dispatch(resume);
84728473
}
84738474
const after = yield* orchestrator.getThreadProjection(threadId);
8474-
if (scenario === "resume") {
8475+
if (scenario === "resume" || scenario === "resume-started") {
84758476
assert.deepInclude(after.messages.at(-1), {
84768477
text: "Continue where you left off.",
84778478
createdBy: "agent",
84788479
creationSource: "server",
84798480
});
84808481
const continuation = after.runs.at(-1)!;
84818482
assert.equal(continuation.status, "starting");
8483+
if (scenario === "resume-started") {
8484+
const startedAt = yield* DateTime.now;
8485+
yield* events.write({
8486+
events: [
8487+
{
8488+
id: EventId.make("recovery:continuation-started:resume-started"),
8489+
type: "run.updated",
8490+
threadId,
8491+
runId: continuation.id,
8492+
occurredAt: startedAt,
8493+
payload: { ...continuation, status: "running", startedAt },
8494+
},
8495+
],
8496+
});
8497+
}
84828498
yield* (yield* ProviderRuntimeRecoveryService.ProviderRuntimeRecoveryService).reconcile(
84838499
"startup",
84848500
);
@@ -8490,19 +8506,28 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
84908506
true,
84918507
DateTime.toEpochMillis(yield* DateTime.now),
84928508
);
8493-
assert(retry?.type === "message.dispatch");
8494-
assert.notEqual(retry.commandId, resume!.commandId);
8495-
yield* orchestrator.dispatch(retry);
8496-
yield* orchestrator.dispatch(retry);
8497-
assert.deepEqual(
8498-
(yield* orchestrator.getThreadProjection(threadId)).runs.map((row) => row.status),
8499-
["failed", "cancelled", "starting"],
8500-
);
8509+
if (scenario === "resume-started") {
8510+
assert.isNull(retry);
8511+
assert.equal(
8512+
restartedShell.limitRecovery?.requestId,
8513+
after.thread.limitRecovery?.requestId,
8514+
);
8515+
} else {
8516+
assert(retry?.type === "message.dispatch");
8517+
assert.notEqual(retry.commandId, resume!.commandId);
8518+
yield* orchestrator.dispatch(retry);
8519+
yield* orchestrator.dispatch(retry);
8520+
assert.deepEqual(
8521+
(yield* orchestrator.getThreadProjection(threadId)).runs.map((row) => row.status),
8522+
["failed", "cancelled", "starting"],
8523+
);
8524+
}
85018525
}
85028526
assert.lengthOf(
85038527
after.runs,
85048528
before.runs.length +
85058529
(scenario === "resume" ||
8530+
scenario === "resume-started" ||
85068531
stopBeforeArm ||
85078532
scenario === "queued-resume" ||
85088533
scenario === "independent-patches" ||
@@ -8514,6 +8539,7 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
85148539
after.messages,
85158540
before.messages.length +
85168541
(scenario === "resume" ||
8542+
scenario === "resume-started" ||
85178543
stopBeforeArm ||
85188544
scenario === "queued-resume" ||
85198545
scenario === "independent-patches" ||

‎packages/contracts/src/orchestrationV2.ts‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -692,7 +692,6 @@ export const OrchestrationV2Run = Schema.Struct({
692692
checkpointId: Schema.NullOr(CheckpointId),
693693
contextHandoffId: Schema.NullOr(ContextHandoffId),
694694
restartContinuationOfRunId: Schema.optional(RunId),
695-
usageLimitContinuationOfRunId: Schema.optional(RunId),
696695
delegatedTaskId: Schema.optional(NodeId),
697696
workStartedAt: Schema.optional(Schema.DateTimeUtc),
698697
restartCancelledBackgroundWork: Schema.optional(

0 commit comments

Comments
 (0)