Skip to content

Commit 0664044

Browse files
committed
fix(server): usage-limit recovery survives notification wakes that did no work
A Claude thread that hits its limit while background subagents run gets one root run per failed subagent, each started by a notification and completed within milliseconds. The newest of those became the thread's latest executed run, so the limited run dropped out of the recovery candidates and the thread was never armed or resumed (or, if already armed, never resumed at the reset). A run that a notification started and that completed before the newest usage-limit failure resets no longer replaces that failure as the latest run. The SQL shell and candidate queries, the shared usageLimitBlockedRun used by the server and clients, arming, manual resume, and Stop all use that rule. Fixes #16815
1 parent 4b87a45 commit 0664044

7 files changed

Lines changed: 264 additions & 18 deletions

File tree

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 14 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1222,7 +1222,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
12221222
"contextTransfers",
12231223
],
12241224
{
1225-
turnItemTypes: ["user_message", "error"],
1225+
turnItemTypes: ["user_message", "error", "notification"],
12261226
messageRoles: ["user"],
12271227
omitSettledLeafNodes: true,
12281228
// Commands here read only undelivered or active tasks; a dispatcher thread keeps
@@ -3449,7 +3449,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
34493449
const projection = yield* loadProjectionForCommand(
34503450
command,
34513451
["runs", "runtimeRequests", "turnItems"],
3452-
{ turnItemTypes: ["error"] },
3452+
{ turnItemTypes: ["user_message", "error", "notification"] },
34533453
);
34543454
const run = usageLimitBlockedRun(projection.runs, projection.turnItems, null);
34553455
const failure = latestRootProviderFailure(run, projection.turnItems);
@@ -5477,7 +5477,10 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
54775477
source === undefined ||
54785478
(source.status !== "interrupted" &&
54795479
!(source.status === "failed" && limited?.class === "usage_limit")) ||
5480-
latestExecutedRun(projection.runs)?.id !== source.id ||
5480+
(
5481+
usageLimitBlockedRun(projection.runs, projection.turnItems, null) ??
5482+
latestExecutedRun(projection.runs)
5483+
)?.id !== source.id ||
54815484
projection.thread.archivedAt !== null ||
54825485
projection.thread.deletedAt !== null ||
54835486
projection.runtimeRequests.some((request) => request.status === "pending")
@@ -10845,14 +10848,14 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1084510848
const { runs, subagents } = input.projection;
1084610849
let { thread } = input.projection;
1084710850
let recovery = thread.limitRecovery;
10848-
const latest = latestExecutedRun(runs);
10851+
const { turnItems } = yield* projectionStore
10852+
.getThreadRecords(thread.id, ["turnItems"], {
10853+
turnItemTypes: ["user_message", "error", "notification"],
10854+
})
10855+
.pipe(mapDispatchError(input.command));
10856+
// Stop must also disarm a limited run that completed notification wakes hide.
10857+
const latest = usageLimitBlockedRun(runs, turnItems, null) ?? latestExecutedRun(runs);
1084910858
if (latest?.status === "failed") {
10850-
const { turnItems } = yield* projectionStore
10851-
.getThreadRecords(thread.id, ["turnItems"], {
10852-
turnItemTypes: ["error"],
10853-
turnItemRunIds: [latest.id],
10854-
})
10855-
.pipe(mapDispatchError(input.command));
1085610859
const failure = latestRootProviderFailure(latest, turnItems);
1085710860
if (
1085810861
failure?.class === "usage_limit" &&
@@ -13346,7 +13349,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1334613349
const projection = yield* loadProjectionForCommand(
1334713350
command,
1334813351
["runs", "turnItems", "providerSessions", "messages"],
13349-
{ turnItemTypes: ["error"], messageRoles: ["user"] },
13352+
{ turnItemTypes: ["user_message", "error", "notification"], messageRoles: ["user"] },
1335013353
);
1335113354
if (projection.thread.archivedAt !== null || projection.thread.deletedAt !== null) {
1335213355
return yield* new OrchestratorDispatchError({

‎apps/server/src/orchestration-v2/ProjectionStore.test.ts‎

Lines changed: 65 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5640,7 +5640,71 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => {
56405640
},
56415641
});
56425642
yield* assertSummary("Plan limit reached.", "usage_limit");
5643+
// A subagent that died on the same limit starts a notification wake that completes
5644+
// before the reset. It did no work, so the failed run stays the latest.
5645+
const wakeRunId = RunId.make("run:limit-shell:wake");
5646+
const wakeNodeId = NodeId.make("node:limit-shell:wake");
5647+
yield* store.apply({
5648+
id: EventId.make("event:limit-shell:wake"),
5649+
type: "run.created",
5650+
threadId,
5651+
runId: wakeRunId,
5652+
nodeId: wakeNodeId,
5653+
driver,
5654+
providerInstanceId,
5655+
occurredAt: now,
5656+
payload: {
5657+
...original,
5658+
id: wakeRunId,
5659+
ordinal: original.ordinal + 4,
5660+
rootNodeId: wakeNodeId,
5661+
userMessageId: MessageId.make("message:limit-shell:wake"),
5662+
status: "completed",
5663+
startedAt: DateTime.add(now, { minutes: 1 }),
5664+
completedAt: DateTime.add(now, { minutes: 1 }),
5665+
},
5666+
});
5667+
yield* store.apply({
5668+
id: EventId.make("event:limit-shell:wake-notification"),
5669+
type: "turn-item.updated",
5670+
threadId,
5671+
runId: wakeRunId,
5672+
nodeId: wakeNodeId,
5673+
driver,
5674+
occurredAt: now,
5675+
payload: {
5676+
id: TurnItemId.make("limit-shell:wake-notification"),
5677+
threadId,
5678+
runId: wakeRunId,
5679+
nodeId: wakeNodeId,
5680+
providerThreadId: null,
5681+
providerTurnId: null,
5682+
nativeItemRef: null,
5683+
parentItemId: null,
5684+
ordinal: 4,
5685+
status: "completed",
5686+
title: null,
5687+
startedAt: now,
5688+
completedAt: now,
5689+
updatedAt: now,
5690+
type: "notification",
5691+
source: { kind: "delegated_task", taskIds: [] },
5692+
outcome: "failed",
5693+
summary: "Subagent failed",
5694+
},
5695+
});
5696+
yield* assertSummary("Plan limit reached.", "usage_limit");
5697+
const wakeProjection = yield* store.getThreadProjection(threadId);
5698+
assert.equal(
5699+
ProjectionStore.threadShellFromProjection(wakeProjection).latestRunId,
5700+
original.id,
5701+
);
5702+
assert.equal(
5703+
(yield* store.getShellSnapshot()).threads.find((row) => row.id === threadId)!.latestRunId,
5704+
original.id,
5705+
);
56435706
const sql = yield* SqlClient.SqlClient;
5707+
yield* sql`DELETE FROM orchestration_v2_projection_runs WHERE run_id = ${wakeRunId}`;
56445708
yield* sql`DELETE FROM orchestration_v2_projection_runs WHERE run_id = ${aheadRunId}`;
56455709
yield* sql`DELETE FROM orchestration_v2_projection_runs WHERE run_id = ${queuedRunId}`;
56465710
yield* assertSummary("Plan limit reached.", "usage_limit");
@@ -6282,7 +6346,7 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => {
62826346
(detail) =>
62836347
/^SEARCH (?:item|error) /.test(detail) && !detail.includes("(turn_item_id=?)"),
62846348
);
6285-
assert.lengthOf(errorLookups, marker.includes("blocking") ? 2 : 1, marker);
6349+
assert.lengthOf(errorLookups, marker.includes("blocking") ? 3 : 2, marker);
62866350
for (const errorLookup of errorLookups) {
62876351
assert.include(errorLookup, "turn_items_failed_error_idx");
62886352
assert.include(errorLookup, "(thread_id=? AND run_id=? AND node_id=?)");

‎apps/server/src/orchestration-v2/ProjectionStore.ts‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3967,6 +3967,50 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
39673967
),
39683968
);
39693969

3970+
// The newest failed run of each thread and its root error, for isLimitedWake.
3971+
const latestFailureJoins = sql`
3972+
LEFT JOIN orchestration_v2_projection_runs latest_failure ON latest_failure.run_id = (
3973+
SELECT failed.run_id FROM orchestration_v2_projection_runs failed
3974+
WHERE failed.thread_id = t.thread_id AND failed.status = 'failed'
3975+
ORDER BY failed.completed_at IS NULL DESC, failed.completed_at DESC,
3976+
failed.ordinal DESC, failed.run_id DESC
3977+
LIMIT 1
3978+
)
3979+
LEFT JOIN orchestration_v2_projection_turn_items latest_failure_error
3980+
ON latest_failure_error.turn_item_id = (
3981+
SELECT error.turn_item_id FROM orchestration_v2_projection_turn_items error
3982+
WHERE error.thread_id = t.thread_id
3983+
AND error.run_id = latest_failure.run_id
3984+
AND error.node_id IS json_extract(latest_failure.payload_json, '$.rootNodeId')
3985+
AND error.type = 'error' AND error.status = 'failed'
3986+
ORDER BY error.updated_at DESC, error.ordinal DESC, error.turn_item_id DESC
3987+
LIMIT 1
3988+
)
3989+
`;
3990+
// Mirrors the wake rule of usageLimitBlockedRun: a run that a notification started and
3991+
// that completed before the newest usage-limit failure resets did no work, so it must
3992+
// not replace that failure as the latest run.
3993+
const isLimitedWake = (alias: string) => {
3994+
const run = sql.literal(alias);
3995+
return sql`(
3996+
${run}.status = 'completed'
3997+
AND COALESCE(
3998+
json_extract(latest_failure_error.payload_json, '$.failure.class') = 'usage_limit'
3999+
AND julianday(${run}.completed_at) <= julianday(
4000+
json_extract(latest_failure_error.payload_json, '$.failure.resetAt')
4001+
), 0
4002+
) AND (
4003+
SELECT wake.type FROM orchestration_v2_projection_turn_items wake
4004+
WHERE wake.run_id = ${run}.run_id
4005+
AND wake.node_id IS NOT NULL
4006+
AND wake.node_id IS json_extract(${run}.payload_json, '$.rootNodeId')
4007+
AND wake.type IN ('user_message', 'notification')
4008+
ORDER BY wake.ordinal
4009+
LIMIT 1
4010+
) IS 'notification'
4011+
)`;
4012+
};
4013+
39704014
const getLimitRecoveryCandidates = Effect.fn("ProjectionStore.getLimitRecoveryCandidates")(
39714015
function* (options: Parameters<ProjectionStoreV2Shape["getLimitRecoveryCandidates"]>[0]) {
39724016
const rows = yield* sql<{
@@ -3990,6 +4034,7 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
39904034
LIMIT 1
39914035
) AS last_error
39924036
FROM orchestration_v2_projection_threads t
4037+
${latestFailureJoins}
39934038
INNER JOIN orchestration_v2_projection_runs r ON r.run_id = (
39944039
SELECT latest.run_id FROM orchestration_v2_projection_runs latest
39954040
WHERE latest.thread_id = t.thread_id
@@ -3998,6 +4043,7 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
39984043
latest.status = 'cancelled'
39994044
AND json_extract(latest.payload_json, '$.startedAt') IS NULL
40004045
)
4046+
AND NOT ${isLimitedWake("latest")}
40014047
-- latestExecutedRun: the run that ended last (runRanAfter).
40024048
ORDER BY latest.completed_at IS NULL DESC, latest.completed_at DESC,
40034049
latest.ordinal DESC, latest.run_id DESC
@@ -6288,6 +6334,7 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
62886334
ORDER BY candidate.ordinal DESC, candidate.run_id DESC
62896335
LIMIT 1
62906336
)
6337+
${latestFailureJoins}
62916338
LEFT JOIN orchestration_v2_projection_runs blocked ON blocked.run_id = (
62926339
SELECT candidate.run_id
62936340
FROM orchestration_v2_projection_runs candidate
@@ -6297,6 +6344,7 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
62976344
candidate.status = 'cancelled'
62986345
AND json_extract(candidate.payload_json, '$.startedAt') IS NULL
62996346
)
6347+
AND NOT ${isLimitedWake("candidate")}
63006348
-- latestExecutedRun: the run that ended last (runRanAfter).
63016349
ORDER BY candidate.completed_at IS NULL DESC, candidate.completed_at DESC,
63026350
candidate.ordinal DESC, candidate.run_id DESC

‎apps/server/src/orchestration-v2/runtimeLayer.test.ts‎

Lines changed: 63 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8004,9 +8004,11 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
80048004
"independent-patches",
80058005
"expired-snooze",
80068006
"wake",
8007+
"wake-hidden",
80078008
"stop",
80088009
"stop-before-arm",
80098010
"stop-before-arm-mismatched-recovery",
8011+
"stop-before-arm-wake-hidden",
80108012
] as const)("guards a scheduled usage-limit continuation against %s", (scenario) =>
80118013
Effect.gen(function* () {
80128014
const orchestrator = yield* Orchestrator.OrchestratorV2;
@@ -8110,6 +8112,61 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
81108112
},
81118113
],
81128114
});
8115+
const hiddenByWake = scenario === "wake-hidden" || scenario === "stop-before-arm-wake-hidden";
8116+
if (hiddenByWake) {
8117+
// A subagent that died on the same limit starts a notification wake that completes
8118+
// before the reset. The failed run stays the latest, so it is armed and resumes.
8119+
const wakeRunId = RunId.make(`recovery:wake-run:${scenario}`);
8120+
const wakeNodeId = NodeId.make(`recovery:wake-node:${scenario}`);
8121+
yield* events.write({
8122+
events: [
8123+
{
8124+
id: EventId.make(`recovery:wake-run-created:${scenario}`),
8125+
type: "run.created",
8126+
threadId,
8127+
runId: wakeRunId,
8128+
providerInstanceId: modelSelection.instanceId,
8129+
occurredAt: now,
8130+
payload: {
8131+
...run,
8132+
id: wakeRunId,
8133+
ordinal: run.ordinal + 1,
8134+
rootNodeId: wakeNodeId,
8135+
userMessageId: MessageId.make(`recovery:wake-message:${scenario}`),
8136+
status: "completed",
8137+
startedAt: now,
8138+
completedAt: now,
8139+
},
8140+
},
8141+
{
8142+
id: EventId.make(`recovery:wake-notification:${scenario}`),
8143+
type: "turn-item.updated",
8144+
threadId,
8145+
occurredAt: now,
8146+
payload: {
8147+
id: TurnItemId.make(`recovery:wake-notification:${scenario}`),
8148+
type: "notification",
8149+
threadId,
8150+
runId: wakeRunId,
8151+
nodeId: wakeNodeId,
8152+
providerThreadId: null,
8153+
providerTurnId: null,
8154+
nativeItemRef: null,
8155+
parentItemId: null,
8156+
ordinal: 1,
8157+
status: "completed",
8158+
title: null,
8159+
startedAt: now,
8160+
completedAt: now,
8161+
updatedAt: now,
8162+
source: { kind: "delegated_task", taskIds: [] },
8163+
outcome: "failed",
8164+
summary: "Subagent failed",
8165+
},
8166+
},
8167+
],
8168+
});
8169+
}
81138170
if (scenario === "queued-resume") {
81148171
const queuedRun = projection.runs[1]!;
81158172
yield* events.write({
@@ -8163,7 +8220,9 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
81638220
limitRecovery: arm.limitRecovery,
81648221
};
81658222
const stopBeforeArm =
8166-
scenario === "stop-before-arm" || scenario === "stop-before-arm-mismatched-recovery";
8223+
scenario === "stop-before-arm" ||
8224+
scenario === "stop-before-arm-mismatched-recovery" ||
8225+
scenario === "stop-before-arm-wake-hidden";
81678226
if (stopBeforeArm) {
81688227
if (scenario === "stop-before-arm-mismatched-recovery") {
81698228
const current = yield* orchestrator.getThreadProjection(threadId);
@@ -8342,7 +8401,7 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
83428401
});
83438402
assert.lengthOf(
83448403
(yield* orchestrator.getThreadProjection(threadId)).runs,
8345-
scenario === "queued-resume" ? 2 : 1,
8404+
scenario === "queued-resume" || hiddenByWake ? 2 : 1,
83468405
);
83478406
yield* TestClock.adjust("1 minute");
83488407
const resume = limitRecoveryCommand(
@@ -8475,6 +8534,7 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
84758534
after.runs,
84768535
before.runs.length +
84778536
(scenario === "resume" ||
8537+
scenario === "wake-hidden" ||
84788538
stopBeforeArm ||
84798539
scenario === "queued-resume" ||
84808540
scenario === "independent-patches" ||
@@ -8486,6 +8546,7 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
84868546
after.messages,
84878547
before.messages.length +
84888548
(scenario === "resume" ||
8549+
scenario === "wake-hidden" ||
84898550
stopBeforeArm ||
84908551
scenario === "queued-resume" ||
84918552
scenario === "independent-patches" ||

‎packages/client-runtime/src/state/threadExecution.test.ts‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -178,6 +178,35 @@ describe("thread execution presentation", () => {
178178
lastErrorClass: null,
179179
});
180180
expect(deriveResumableRunId(providerError, deriveThreadRuntime(providerError))).toBeNull();
181+
182+
// A notification wake that completes before the reset did no work and keeps the limit
183+
// in place. One that completes after the reset is a real run and clears it.
184+
const resetAt = "2026-07-28T11:00:00.000Z";
185+
const limited = { ...item, failure: { ...item.failure, resetAt } };
186+
const wake = {
187+
...run("wake", 2, "completed"),
188+
rootNodeId: NodeId.make("wake-root"),
189+
completedAt: DateTime.makeUnsafe("2026-07-28T10:30:00.000Z"),
190+
};
191+
const notification = {
192+
...item,
193+
id: TurnItemId.make("wake-notification"),
194+
runId: wake.id,
195+
nodeId: wake.rootNodeId,
196+
type: "notification" as const,
197+
status: "completed" as const,
198+
source: { kind: "delegated_task" as const, taskIds: [] },
199+
outcome: "failed" as const,
200+
summary: "Subagent failed",
201+
};
202+
const hidden = { ...v2Projection, runs: [failed, wake], turnItems: [limited, notification] };
203+
expect(deriveThreadRuntime(hidden)).toMatchObject({
204+
status: "failed",
205+
lastErrorClass: "usage_limit",
206+
});
207+
expect(deriveResumableRunId(hidden, deriveThreadRuntime(hidden))).toBe(failed.id);
208+
const afterReset = { ...wake, completedAt: DateTime.makeUnsafe("2026-07-28T11:30:00.000Z") };
209+
expect(usageLimitBlockedRun([failed, afterReset], hidden.turnItems, null)).toBeNull();
181210
});
182211

183212
it("keeps live activity attached to an executing run when a newer run is queued", () => {

‎packages/client-runtime/src/state/threadExecution.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import {
33
latestRootProviderFailure,
44
latestUnheldRun,
55
threadErrorSummary,
6+
usageLimitBlockedRun,
67
usageLimitRunPresentedAsLatest,
78
} from "@t3tools/shared/orchestrationV2ThreadError";
89
import {
@@ -217,7 +218,9 @@ export function deriveResumableRunId(
217218
projection: Pick<ThreadStructure, "runs" | "turnItems">,
218219
runtime: Pick<ThreadStructureRuntime, "lastErrorClass"> | null,
219220
): RunId | null {
220-
const run = latestExecutedRun(projection.runs);
221+
const run =
222+
usageLimitBlockedRun(projection.runs, projection.turnItems, null) ??
223+
latestExecutedRun(projection.runs);
221224
if (run?.status === "interrupted") return run.id;
222225
return run?.status === "failed" &&
223226
runtime?.lastErrorClass === "usage_limit" &&

0 commit comments

Comments
 (0)