Skip to content

Commit 99b4119

Browse files
chledowskiclaude
andcommitted
fix(server): a redeemed reset continues only threads whose limit it refilled
Review of the first commit found that a redemption continued every waiting thread of the account, whatever limit the credit refilled, and that a client could set the mark itself. - Redemption moves from the ws.ts handler into a ResetCredits service. It reads the account's limits after the redemption and leaves alone threads due at or after the reset of a limit that is still used up. An unconfirmed redemption continues nothing. A retry answered with alreadyRedeemed counts as the reset. A caller that disconnects once the credit is spent no longer interrupts the follow-up. - The mark is set by a server-only command, thread.limit-recovery.reset, so it is off the client's metadata update. - Each redemption uses its own command id, so a recovery refused once can be marked by a later redemption. A pending approval or question no longer refuses the mark. Co-Authored-By: Claude Fable 5.2 <[email protected]>
1 parent 620e2d8 commit 99b4119

12 files changed

Lines changed: 558 additions & 238 deletions

File tree

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 77 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -390,6 +390,8 @@ export interface OrchestratorV2Shape {
390390
readonly providerInstanceIds: ReadonlyArray<ProviderInstanceId>;
391391
/** A run that failed later hit a limit the reset did not clear. */
392392
readonly failedBefore: DateTime.Utc;
393+
/** A recovery that waits until this time or later waits on a limit the reset did not clear. */
394+
readonly resetBefore?: DateTime.Utc;
393395
}) => Effect.Effect<void, OrchestratorV2Error>;
394396
readonly searchThreadStream: (
395397
input: OrchestrationV2SearchThreadInput,
@@ -590,6 +592,7 @@ function commandThreadId(command: OrchestrationV2ServerCommand): ThreadId {
590592
case "thread.pull-request-watch.sync":
591593
case "thread.pull-request.sync":
592594
case "thread.title.regeneration.complete":
595+
case "thread.limit-recovery.reset":
593596
case "thread.project.adopt-folder":
594597
case "thread.runtime-mode.set":
595598
case "thread.interaction-mode.set":
@@ -4703,6 +4706,71 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
47034706
},
47044707
);
47054708

4709+
const dispatchLimitRecoveryReset = Effect.fn("orchestrationV2.dispatch.limitRecoveryReset")(
4710+
function* (
4711+
command: Extract<
4712+
OrchestrationV2ServerCommand,
4713+
{ readonly type: "thread.limit-recovery.reset" }
4714+
>,
4715+
events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
4716+
) {
4717+
const projection = yield* projectionStore
4718+
.getThreadRecords(command.threadId, ["runs", "turnItems"], { turnItemTypes: ["error"] })
4719+
.pipe(
4720+
Effect.mapError(
4721+
(cause) => new OrchestratorProjectionError({ threadId: command.threadId, cause }),
4722+
),
4723+
);
4724+
const thread = projection.thread;
4725+
const recovery = thread.limitRecovery;
4726+
const run = usageLimitBlockedRun(projection.runs, projection.turnItems, null);
4727+
const now = yield* DateTime.now;
4728+
if (
4729+
recovery?.runId !== command.runId ||
4730+
recovery.resetAt !== command.resetAt ||
4731+
recovery.resetEarly === true ||
4732+
!(recovery.autoResume || recovery.snooze) ||
4733+
Date.parse(recovery.resetAt) <= DateTime.toEpochMillis(now) ||
4734+
run?.id !== recovery.runId ||
4735+
latestRootProviderFailure(run, projection.turnItems)?.resetAt !== recovery.resetAt ||
4736+
thread.archivedAt !== null ||
4737+
thread.deletedAt !== null ||
4738+
thread.settledOverride === "settled"
4739+
) {
4740+
return yield* new OrchestratorDispatchError({
4741+
commandId: command.commandId,
4742+
commandType: command.type,
4743+
cause: `Thread ${command.threadId} no longer waits on that limit.`,
4744+
});
4745+
}
4746+
yield* emit(
4747+
events,
4748+
command,
4749+
)({
4750+
type: "thread.metadata-updated",
4751+
threadId: command.threadId,
4752+
providerInstanceId: thread.providerInstanceId,
4753+
occurredAt: now,
4754+
payload: {
4755+
...thread,
4756+
limitRecovery: {
4757+
...recovery,
4758+
snooze: false,
4759+
resetEarly: true,
4760+
requestId: command.commandId,
4761+
},
4762+
// Ends only the snooze this recovery set; one the user chose stays.
4763+
...(recovery.snooze &&
4764+
thread.snoozedUntil != null &&
4765+
DateTime.toEpochMillis(thread.snoozedUntil) === Date.parse(recovery.resetAt)
4766+
? { snoozedUntil: null, snoozedAt: null, lastSnoozeWakeAt: now }
4767+
: {}),
4768+
updatedAt: now,
4769+
},
4770+
});
4771+
},
4772+
);
4773+
47064774
const dispatchThreadProjectAdoptFolder = Effect.fn(
47074775
"orchestrationV2.dispatch.threadProjectAdoptFolder",
47084776
)(function* (
@@ -13664,6 +13732,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1366413732
case "provider-session.detach":
1366513733
yield* dispatchProviderSessionDetach(command, events, effects);
1366613734
break;
13735+
case "thread.limit-recovery.reset":
13736+
yield* dispatchLimitRecoveryReset(command, events);
13737+
break;
1366713738
case "thread.project.adopt-folder":
1366813739
yield* dispatchThreadProjectAdoptFolder(command, events);
1366913740
break;
@@ -14577,15 +14648,17 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1457714648
const waiting = yield* projectionStore
1457814649
.getLimitRecoveriesAwaitingReset({ ...input, now: yield* DateTime.now })
1457914650
.pipe(Effect.mapError(shellProjectionError));
14651+
// A refusal is stored under its command id, so each reset tries under its own.
14652+
const reset = DateTime.toEpochMillis(input.failedBefore);
1458014653
yield* Effect.forEach(
1458114654
waiting,
1458214655
({ threadId, limitRecovery: { runId, resetAt } }) =>
1458314656
dispatchWithReceipt({
14584-
type: "thread.metadata.update",
14585-
commandId: CommandId.make(`limit-reset:${threadId}:${runId}:${Date.parse(resetAt)}`),
14657+
type: "thread.limit-recovery.reset",
14658+
commandId: CommandId.make(`limit-reset:${threadId}:${runId}:${reset}`),
1458614659
threadId,
14587-
// Ends only the snooze this recovery set; one the user chose stays.
14588-
limitRecovery: { runId, resetAt, snooze: false, resetEarly: true },
14660+
runId,
14661+
resetAt,
1458914662
}).pipe(
1459014663
// A thread that moved on since the read is refused and keeps its own schedule.
1459114664
Effect.catchCause((cause) =>

‎apps/server/src/orchestration-v2/ProjectionStore.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -569,6 +569,7 @@ export interface ProjectionStoreV2Shape {
569569
readonly now: DateTime.Utc;
570570
readonly providerInstanceIds: ReadonlyArray<ProviderInstanceId>;
571571
readonly failedBefore: DateTime.Utc;
572+
readonly resetBefore?: DateTime.Utc;
572573
}) => Effect.Effect<ReadonlyArray<ProjectionAwaitedLimitRecovery>, ProjectionStoreV2Error>;
573574
readonly getSettlementCandidates: (
574575
threadId?: ThreadId,
@@ -4249,6 +4250,7 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
42494250
options: Parameters<ProjectionStoreV2Shape["getLimitRecoveriesAwaitingReset"]>[0],
42504251
) {
42514252
if (options.providerInstanceIds.length === 0) return [];
4253+
const resetBefore = options.resetBefore ? DateTime.formatIso(options.resetBefore) : null;
42524254
const rows = yield* sql<{ readonly payload_json: string }>`
42534255
SELECT t.payload_json
42544256
FROM orchestration_v2_projection_threads t
@@ -4261,6 +4263,10 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
42614263
AND r.provider_instance_id IN ${sql.in(options.providerInstanceIds)}
42624264
AND julianday(COALESCE(r.completed_at, json_extract(t.payload_json, '$.updatedAt'))) <= julianday(${DateTime.formatIso(options.failedBefore)})
42634265
AND julianday(json_extract(t.payload_json, '$.limitRecovery.resetAt')) > julianday(${DateTime.formatIso(options.now)})
4266+
AND (
4267+
${resetBefore} IS NULL
4268+
OR julianday(json_extract(t.payload_json, '$.limitRecovery.resetAt')) < julianday(${resetBefore})
4269+
)
42644270
AND json_extract(t.payload_json, '$.limitRecovery.resetEarly') IS NOT 1
42654271
AND (
42664272
json_extract(t.payload_json, '$.limitRecovery.autoResume') = 1

‎apps/server/src/orchestration-v2/runtimeLayer.test.ts‎

Lines changed: 24 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -8651,24 +8651,41 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
86518651
snoozedUntil: DateTime.formatIso(manualWakeAt),
86528652
});
86538653
const waiting = yield* orchestrator.getThreadProjection(threadId);
8654-
// A reset of another account, or one from before this run failed, clears nothing here.
8654+
const instances = { providerInstanceIds: [run.providerInstanceId] };
8655+
// A reset of another account, one from before this run failed, or one that left this
8656+
// run's limit used up clears nothing here.
86558657
yield* orchestrator.recordLimitReset({
86568658
providerInstanceIds: [alternateInstanceId],
86578659
failedBefore: now,
86588660
});
86598661
yield* orchestrator.recordLimitReset({
8660-
providerInstanceIds: [run.providerInstanceId],
8662+
...instances,
86618663
failedBefore: DateTime.subtract(now, { seconds: 1 }),
86628664
});
8665+
yield* orchestrator.recordLimitReset({
8666+
...instances,
8667+
failedBefore: now,
8668+
resetBefore: DateTime.makeUnsafe(resetAt),
8669+
});
8670+
// So does a reset of a limit this thread does not wait on.
8671+
yield* orchestrator
8672+
.dispatch({
8673+
type: "thread.limit-recovery.reset",
8674+
commandId: CommandId.make(`recovery:other-limit:${scenario}`),
8675+
threadId,
8676+
runId: run.id,
8677+
resetAt: DateTime.formatIso(DateTime.add(DateTime.makeUnsafe(resetAt), { hours: 1 })),
8678+
})
8679+
.pipe(Effect.flip);
86638680
assert.deepEqual(
86648681
(yield* orchestrator.getThreadProjection(threadId)).thread,
86658682
waiting.thread,
86668683
);
8667-
const recordReset = orchestrator.recordLimitReset({
8668-
providerInstanceIds: [run.providerInstanceId],
8684+
yield* orchestrator.recordLimitReset({
8685+
...instances,
86698686
failedBefore: now,
8687+
resetBefore: DateTime.add(DateTime.makeUnsafe(resetAt), { seconds: 1 }),
86708688
});
8671-
yield* recordReset;
86728689
const reset = (yield* orchestrator.getThreadProjection(threadId)).thread;
86738690
assert.deepInclude(reset.limitRecovery, { autoResume, snooze: false, resetEarly: true });
86748691
// The reset ends the snooze the recovery set, never one the user chose.
@@ -8705,7 +8722,8 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
87058722
assert(resume?.type === "message.dispatch");
87068723
assert.isBelow(DateTime.toEpochMillis(yield* DateTime.now), Date.parse(resetAt));
87078724
yield* orchestrator.dispatch(resume);
8708-
yield* recordReset;
8725+
// A second redemption, and a repeat of the continuation, start nothing more.
8726+
yield* orchestrator.recordLimitReset({ ...instances, failedBefore: yield* DateTime.now });
87098727
yield* orchestrator.dispatch(resume);
87108728
const resumed = yield* orchestrator.getThreadProjection(threadId);
87118729
assert.lengthOf(resumed.runs, 2);

‎apps/server/src/server.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,7 @@ import * as ProviderInstanceRegistry from "./provider/ProviderInstanceRegistry.t
114114
import * as ProviderAdapterRegistry from "./orchestration-v2/ProviderAdapterRegistry.ts";
115115
import * as ProviderRegistry from "./provider/ProviderRegistry.ts";
116116
import * as ProviderUsageLimitsIngestion from "./provider/ProviderUsageLimitsIngestion.ts";
117+
import * as ResetCredits from "./usage/ResetCredits.ts";
117118
import * as UsageLimitSources from "./usage/UsageLimitSources.ts";
118119
import * as ProjectFaviconResolver from "./project/ProjectFaviconResolver.ts";
119120
import * as T3ProjectFileLoader from "./project/T3ProjectFileLoader.ts";
@@ -682,6 +683,7 @@ const layerRuntimeCoreDependenciesBase = Layer.mergeAll(
682683
ProviderUsageLimitsIngestion.layer,
683684
layerProviderInstallationRefresh,
684685
ProviderInstallation.layer,
686+
ResetCredits.layer,
685687
ReplayMarkers.layer,
686688
Layer.effectDiscard(ProjectSettingsOverrideSweep.sweep),
687689
).pipe(

0 commit comments

Comments
 (0)