Skip to content

Commit 620e2d8

Browse files
chledowskiclaude
andcommitted
fix(server): using a banked usage reset continues threads waiting on that limit
A thread stopped by a usage limit can be scheduled to continue at the limit's reset time and snoozed until then. Spending a banked reset under Usage -> Limits cleared the limit, but the thread kept waiting for the reset time the provider reported when the run failed. A redemption that returns `reset` now marks the armed recoveries of the redeemed account's provider instances as reset early. Their snooze until reset ends, and the recovery worker continues the ones with a scheduled continuation on its next sweep. A snooze the user set stays in place. Co-Authored-By: Claude Fable 5.2 <[email protected]>
1 parent 362ef9a commit 620e2d8

12 files changed

Lines changed: 444 additions & 6 deletions

File tree

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -385,6 +385,12 @@ export interface OrchestratorV2Shape {
385385
readonly threadId: ThreadId;
386386
readonly limitRecovery: OrchestrationV2LimitRecoveryUpdate;
387387
}) => Effect.Effect<void, OrchestratorV2Error>;
388+
/** Marks recoveries waiting on these provider instances as reset early: their snooze ends and continuations start. */
389+
readonly recordLimitReset: (input: {
390+
readonly providerInstanceIds: ReadonlyArray<ProviderInstanceId>;
391+
/** A run that failed later hit a limit the reset did not clear. */
392+
readonly failedBefore: DateTime.Utc;
393+
}) => Effect.Effect<void, OrchestratorV2Error>;
388394
readonly searchThreadStream: (
389395
input: OrchestrationV2SearchThreadInput,
390396
) => Stream.Stream<OrchestrationV2SearchThreadResult, OrchestratorV2Error>;
@@ -4059,6 +4065,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
40594065
autoResume:
40604066
command.limitRecovery.autoResume ?? previousRecovery?.autoResume ?? false,
40614067
snooze: command.limitRecovery.snooze ?? previousRecovery?.snooze ?? false,
4068+
...(previousRecovery?.resetEarly ? { resetEarly: true } : {}),
40624069
requestId: command.commandId,
40634070
};
40644071
return {
@@ -5885,7 +5892,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
58855892
recovery.requestId !== command.usageLimitRecoveryRequestId ||
58865893
recovery.runId !== run.id ||
58875894
recovery.resetAt !== failure.resetAt ||
5888-
Date.parse(recovery.resetAt) > DateTime.toEpochMillis(now) ||
5895+
(recovery.resetEarly !== true &&
5896+
Date.parse(recovery.resetAt) > DateTime.toEpochMillis(now)) ||
58895897
projection.thread.archivedAt !== null ||
58905898
projection.thread.deletedAt !== null ||
58915899
projection.thread.settledOverride === "settled" ||
@@ -14565,6 +14573,28 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1456514573
}),
1456614574
);
1456714575
}),
14576+
recordLimitReset: Effect.fn("OrchestratorV2.recordLimitReset")(function* (input) {
14577+
const waiting = yield* projectionStore
14578+
.getLimitRecoveriesAwaitingReset({ ...input, now: yield* DateTime.now })
14579+
.pipe(Effect.mapError(shellProjectionError));
14580+
yield* Effect.forEach(
14581+
waiting,
14582+
({ threadId, limitRecovery: { runId, resetAt } }) =>
14583+
dispatchWithReceipt({
14584+
type: "thread.metadata.update",
14585+
commandId: CommandId.make(`limit-reset:${threadId}:${runId}:${Date.parse(resetAt)}`),
14586+
threadId,
14587+
// Ends only the snooze this recovery set; one the user chose stays.
14588+
limitRecovery: { runId, resetAt, snooze: false, resetEarly: true },
14589+
}).pipe(
14590+
// A thread that moved on since the read is refused and keeps its own schedule.
14591+
Effect.catchCause((cause) =>
14592+
Effect.logDebug("orchestration-v2.limit-recovery.reset-skipped", { threadId, cause }),
14593+
),
14594+
),
14595+
{ discard: true },
14596+
);
14597+
}),
1456814598
searchThreadStream: (input) =>
1456914599
projectionStore
1457014600
.searchThreadStream(input)

‎apps/server/src/orchestration-v2/ProjectionStore.test.ts‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6348,6 +6348,40 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => {
63486348
now: reset,
63496349
})).find((row) => row.id === threadId)!;
63506350
assert.deepEqual(due.limitRecovery, recovery);
6351+
const awaitingReset = store
6352+
.getLimitRecoveriesAwaitingReset({
6353+
now,
6354+
providerInstanceIds: [original.providerInstanceId],
6355+
failedBefore: now,
6356+
})
6357+
.pipe(Effect.map((rows) => rows.filter((row) => row.threadId === threadId)));
6358+
assert.deepEqual(yield* awaitingReset, [{ threadId, limitRecovery: recovery }]);
6359+
const [armedRow] = yield* sql<{
6360+
payload_json: string;
6361+
}>`SELECT payload_json FROM orchestration_v2_projection_threads WHERE thread_id = ${threadId}`;
6362+
// Nothing waits on a reset once it is recorded, cancelled, archived, settled or deleted.
6363+
for (const change of [
6364+
sql`payload_json = json_set(payload_json, '$.limitRecovery.resetEarly', json('true'))`,
6365+
sql`payload_json = json_set(payload_json, '$.limitRecovery.autoResume', json('false'))`,
6366+
sql`payload_json = json_set(payload_json, '$.archivedAt', ${DateTime.formatIso(now)})`,
6367+
sql`payload_json = json_set(payload_json, '$.settledOverride', 'settled')`,
6368+
sql`deleted_at = ${DateTime.formatIso(now)}`,
6369+
]) {
6370+
yield* sql`UPDATE orchestration_v2_projection_threads SET ${change} WHERE thread_id = ${threadId}`;
6371+
assert.deepEqual(yield* awaitingReset, []);
6372+
yield* sql`UPDATE orchestration_v2_projection_threads
6373+
SET payload_json = ${armedRow!.payload_json}, deleted_at = NULL WHERE thread_id = ${threadId}`;
6374+
}
6375+
yield* sql`UPDATE orchestration_v2_projection_threads
6376+
SET payload_json = json_set(payload_json, '$.limitRecovery.resetEarly', json('true'))
6377+
WHERE thread_id = ${threadId}`;
6378+
assert.isDefined(
6379+
(yield* store.getLimitRecoveryCandidates({ ...recoveryOptions, autoResume: true })).find(
6380+
(row) => row.id === threadId,
6381+
),
6382+
);
6383+
yield* sql`UPDATE orchestration_v2_projection_threads
6384+
SET payload_json = ${armedRow!.payload_json} WHERE thread_id = ${threadId}`;
63516385
yield* sql`INSERT INTO orchestration_v2_projection_runtime_requests
63526386
(runtime_request_id, thread_id, node_id, kind, status, created_at, payload_json)
63536387
VALUES ('limit-shell:pending-request', ${threadId}, ${original.rootNodeId}, 'approval', 'pending', ${DateTime.formatIso(now)}, '{}')`;

‎apps/server/src/orchestration-v2/ProjectionStore.ts‎

Lines changed: 54 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ import type {
2929
OrchestrationV2ConversationMessage,
3030
OrchestrationV2ContextTransfer,
3131
OrchestrationV2DomainEvent,
32+
OrchestrationV2LimitRecovery,
3233
OrchestrationV2OlderSettledCount,
3334
OrchestrationV2OlderSettledCursor,
3435
OrchestrationV2ProjectedTurnItem,
@@ -190,6 +191,11 @@ export type ProjectionLimitRecoveryCandidate = Pick<
190191
| "snoozedUntil"
191192
>;
192193

194+
export interface ProjectionAwaitedLimitRecovery {
195+
readonly threadId: ThreadId;
196+
readonly limitRecovery: OrchestrationV2LimitRecovery;
197+
}
198+
193199
export type ProjectionThreadPullRequests = Pick<
194200
OrchestrationV2AppThread,
195201
"id" | "projectId" | "lineage" | "settledOverride" | "settledAt" | "pullRequests"
@@ -558,6 +564,12 @@ export interface ProjectionStoreV2Shape {
558564
readonly getLimitRecoveryWakeAt: (
559565
now: DateTime.Utc,
560566
) => Effect.Effect<number | null, ProjectionStoreV2Error>;
567+
/** Auto-resume or snooze recoveries not yet reset, for runs these instances failed by `failedBefore`. */
568+
readonly getLimitRecoveriesAwaitingReset: (options: {
569+
readonly now: DateTime.Utc;
570+
readonly providerInstanceIds: ReadonlyArray<ProviderInstanceId>;
571+
readonly failedBefore: DateTime.Utc;
572+
}) => Effect.Effect<ReadonlyArray<ProjectionAwaitedLimitRecovery>, ProjectionStoreV2Error>;
561573
readonly getSettlementCandidates: (
562574
threadId?: ThreadId,
563575
) => Effect.Effect<ReadonlyArray<ProjectionSettlementCandidate>, ProjectionStoreV2Error>;
@@ -4135,7 +4147,10 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
41354147
json_extract(t.payload_json, '$.limitRecovery.runId') IS r.run_id
41364148
AND json_extract(t.payload_json, '$.limitRecovery.resetAt') IS json_extract(item.payload_json, '$.failure.resetAt')
41374149
AND json_extract(t.payload_json, '$.limitRecovery.autoResume') = 1
4138-
AND julianday(json_extract(item.payload_json, '$.failure.resetAt')) <= julianday(${DateTime.formatIso(options.now)})
4150+
AND (
4151+
json_extract(t.payload_json, '$.limitRecovery.resetEarly') = 1
4152+
OR julianday(json_extract(item.payload_json, '$.failure.resetAt')) <= julianday(${DateTime.formatIso(options.now)})
4153+
)
41394154
AND (
41404155
json_extract(t.payload_json, '$.snoozedUntil') IS NULL
41414156
OR julianday(json_extract(t.payload_json, '$.snoozedUntil')) <= julianday(${DateTime.formatIso(options.now)})
@@ -4227,6 +4242,43 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
42274242
Effect.mapError((cause) => new ProjectionStoreSetupError({ cause })),
42284243
);
42294244

4245+
const getLimitRecoveriesAwaitingReset = Effect.fn(
4246+
"ProjectionStore.getLimitRecoveriesAwaitingReset",
4247+
)(
4248+
function* (
4249+
options: Parameters<ProjectionStoreV2Shape["getLimitRecoveriesAwaitingReset"]>[0],
4250+
) {
4251+
if (options.providerInstanceIds.length === 0) return [];
4252+
const rows = yield* sql<{ readonly payload_json: string }>`
4253+
SELECT t.payload_json
4254+
FROM orchestration_v2_projection_threads t
4255+
INNER JOIN orchestration_v2_projection_runs r
4256+
ON r.run_id = json_extract(t.payload_json, '$.limitRecovery.runId')
4257+
WHERE t.deleted_at IS NULL
4258+
AND json_extract(t.payload_json, '$.archivedAt') IS NULL
4259+
AND json_extract(t.payload_json, '$.settledOverride') IS NOT 'settled'
4260+
AND r.status = 'failed'
4261+
AND r.provider_instance_id IN ${sql.in(options.providerInstanceIds)}
4262+
AND julianday(COALESCE(r.completed_at, json_extract(t.payload_json, '$.updatedAt'))) <= julianday(${DateTime.formatIso(options.failedBefore)})
4263+
AND julianday(json_extract(t.payload_json, '$.limitRecovery.resetAt')) > julianday(${DateTime.formatIso(options.now)})
4264+
AND json_extract(t.payload_json, '$.limitRecovery.resetEarly') IS NOT 1
4265+
AND (
4266+
json_extract(t.payload_json, '$.limitRecovery.autoResume') = 1
4267+
OR json_extract(t.payload_json, '$.limitRecovery.snooze') = 1
4268+
)
4269+
ORDER BY t.thread_id
4270+
`;
4271+
const waiting: Array<ProjectionAwaitedLimitRecovery> = [];
4272+
for (const row of rows) {
4273+
const thread = yield* decodeThreadPayload(row.payload_json);
4274+
if (thread.limitRecovery != null)
4275+
waiting.push({ threadId: thread.id, limitRecovery: thread.limitRecovery });
4276+
}
4277+
return waiting;
4278+
},
4279+
Effect.mapError((cause) => new ProjectionStoreSetupError({ cause })),
4280+
);
4281+
42304282
const getRecoveryThreadIds = Effect.fn("ProjectionStore.getRecoveryThreadIds")(
42314283
function* (
42324284
kind: ProjectionRecoveryKind,
@@ -7657,6 +7709,7 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
76577709
getProviderControlContext: readOnReader(getProviderControlContext),
76587710
getLimitRecoveryCandidates: readOnReader(getLimitRecoveryCandidates),
76597711
getLimitRecoveryWakeAt: readOnReader(getLimitRecoveryWakeAt),
7712+
getLimitRecoveriesAwaitingReset: readOnReader(getLimitRecoveriesAwaitingReset),
76607713
getRecoveryThreadIds: readOnReader(getRecoveryThreadIds),
76617714
getUnreadableThreadIds: readOnReader(getUnreadableThreadIds),
76627715
getThreadSnapshot: readOnReader(getThreadSnapshot),

‎apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -211,6 +211,8 @@ it.effect(
211211
apply: () => Effect.void,
212212
getLimitRecoveryCandidates: () => Effect.die("unused getLimitRecoveryCandidates"),
213213
getLimitRecoveryWakeAt: () => Effect.die("unused getLimitRecoveryWakeAt"),
214+
getLimitRecoveriesAwaitingReset: () =>
215+
Effect.die("unused getLimitRecoveriesAwaitingReset"),
214216
getOpenRemoteDelegatedTasks: Effect.die("unused getOpenRemoteDelegatedTasks"),
215217
getAgentAwarenessSnapshot: () => Effect.die("unused awareness snapshot"),
216218
getShellSnapshot: () => Effect.die("unused getShellSnapshot"),

‎apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ export function limitRecoveryCommand(
5656
}
5757
if (
5858
!recovery.autoResume ||
59-
resetMs > nowMs ||
59+
(recovery.resetEarly !== true && resetMs > nowMs) ||
6060
(thread.snoozedUntil != null && DateTime.toEpochMillis(thread.snoozedUntil) > nowMs)
6161
)
6262
return null;

‎apps/server/src/orchestration-v2/runtimeLayer.test.ts‎

Lines changed: 85 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8407,6 +8407,9 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
84078407
"stop",
84088408
"stop-before-arm",
84098409
"stop-before-arm-mismatched-recovery",
8410+
"early-reset",
8411+
"early-reset-snooze-only",
8412+
"early-reset-manual-snooze",
84108413
] as const)("guards a scheduled usage-limit continuation against %s", (scenario) =>
84118414
Effect.gen(function* () {
84128415
const orchestrator = yield* Orchestrator.OrchestratorV2;
@@ -8553,8 +8556,12 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
85538556
assert.isNull(current.thread.snoozedUntil);
85548557
return;
85558558
}
8556-
const snooze = scenario === "wake" || scenario === "cancel-resume-keep-snooze";
8557-
const autoResume = scenario !== "wake";
8559+
const snooze =
8560+
scenario === "wake" ||
8561+
scenario === "cancel-resume-keep-snooze" ||
8562+
scenario === "early-reset" ||
8563+
scenario === "early-reset-snooze-only";
8564+
const autoResume = scenario !== "wake" && scenario !== "early-reset-snooze-only";
85588565
const arm = limitRecoveryCommand(shell, autoResume, DateTime.toEpochMillis(now), snooze);
85598566
assert(arm?.type === "thread.metadata.update" && arm.limitRecovery != null);
85608567
const automaticArm = {
@@ -8630,6 +8637,82 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
86308637
});
86318638
if (snooze)
86328639
assert.equal(DateTime.toEpochMillis(armedShell.snoozedUntil!), Date.parse(resetAt));
8640+
if (
8641+
scenario === "early-reset" ||
8642+
scenario === "early-reset-snooze-only" ||
8643+
scenario === "early-reset-manual-snooze"
8644+
) {
8645+
const manualWakeAt = DateTime.add(now, { seconds: 30 });
8646+
if (scenario === "early-reset-manual-snooze")
8647+
yield* orchestrator.dispatch({
8648+
type: "thread.snooze",
8649+
commandId: CommandId.make(`recovery:manual-snooze:${scenario}`),
8650+
threadId,
8651+
snoozedUntil: DateTime.formatIso(manualWakeAt),
8652+
});
8653+
const waiting = yield* orchestrator.getThreadProjection(threadId);
8654+
// A reset of another account, or one from before this run failed, clears nothing here.
8655+
yield* orchestrator.recordLimitReset({
8656+
providerInstanceIds: [alternateInstanceId],
8657+
failedBefore: now,
8658+
});
8659+
yield* orchestrator.recordLimitReset({
8660+
providerInstanceIds: [run.providerInstanceId],
8661+
failedBefore: DateTime.subtract(now, { seconds: 1 }),
8662+
});
8663+
assert.deepEqual(
8664+
(yield* orchestrator.getThreadProjection(threadId)).thread,
8665+
waiting.thread,
8666+
);
8667+
const recordReset = orchestrator.recordLimitReset({
8668+
providerInstanceIds: [run.providerInstanceId],
8669+
failedBefore: now,
8670+
});
8671+
yield* recordReset;
8672+
const reset = (yield* orchestrator.getThreadProjection(threadId)).thread;
8673+
assert.deepInclude(reset.limitRecovery, { autoResume, snooze: false, resetEarly: true });
8674+
// The reset ends the snooze the recovery set, never one the user chose.
8675+
if (scenario === "early-reset-manual-snooze")
8676+
assert.equal(
8677+
DateTime.toEpochMillis(reset.snoozedUntil!),
8678+
DateTime.toEpochMillis(manualWakeAt),
8679+
);
8680+
else assert.isNull(reset.snoozedUntil);
8681+
const projections = yield* ProjectionStore.ProjectionStoreV2;
8682+
const sweep = Effect.gen(function* () {
8683+
const sweptAt = yield* DateTime.now;
8684+
const candidate = (yield* projections.getLimitRecoveryCandidates({
8685+
now: sweptAt,
8686+
autoResume: false,
8687+
snooze: false,
8688+
})).find((row) => row.id === threadId);
8689+
return candidate === undefined
8690+
? null
8691+
: limitRecoveryCommand(candidate, false, DateTime.toEpochMillis(sweptAt));
8692+
});
8693+
if (scenario !== "early-reset") {
8694+
assert.isNull(yield* sweep);
8695+
if (scenario === "early-reset-manual-snooze") yield* TestClock.adjust("30 seconds");
8696+
else
8697+
yield* orchestrator.dispatch({
8698+
type: "thread.metadata.update",
8699+
commandId: CommandId.make(`recovery:resume-after-reset:${scenario}`),
8700+
threadId,
8701+
limitRecovery: { runId: run.id, resetAt, autoResume: true },
8702+
});
8703+
}
8704+
const resume = yield* sweep;
8705+
assert(resume?.type === "message.dispatch");
8706+
assert.isBelow(DateTime.toEpochMillis(yield* DateTime.now), Date.parse(resetAt));
8707+
yield* orchestrator.dispatch(resume);
8708+
yield* recordReset;
8709+
yield* orchestrator.dispatch(resume);
8710+
const resumed = yield* orchestrator.getThreadProjection(threadId);
8711+
assert.lengthOf(resumed.runs, 2);
8712+
assert.equal(resumed.runs[1]!.status, "starting");
8713+
assert.equal(resumed.runs[1]!.usageLimitContinuationOfRunId, run.id);
8714+
return;
8715+
}
86338716
if (scenario === "cancel-resume-keep-snooze") {
86348717
yield* TestClock.adjust("10 seconds");
86358718
yield* orchestrator.dispatch({

0 commit comments

Comments
 (0)