Skip to content

Commit 016489f

Browse files
only21milclaude
andcommitted
fix(server): early recovery ends the recovery's own snooze
With Snooze and Auto-resume both on, the snooze until the reset blocked an early resume. The recovery's own snooze now yields to it; a snooze set from the sidebar still holds the thread. Reported by Cujuju, ported from Cujuju/t3code@e43d5f4971. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent cf6c162 commit 016489f

4 files changed

Lines changed: 64 additions & 26 deletions

File tree

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -811,6 +811,22 @@ function limitRecoveryWaitsForReset(
811811
return previous?.runId === runId && previous.waitForReset === true;
812812
}
813813

814+
// A snooze until the recovery's reset ends when the thread resumes early; one the user set stays.
815+
export function snoozeHoldsLimitRecovery(
816+
thread: {
817+
readonly snoozedUntil?: DateTime.Utc | null | undefined;
818+
readonly limitRecovery?: OrchestrationV2LimitRecovery | null | undefined;
819+
},
820+
nowMs: number,
821+
): boolean {
822+
if (thread.snoozedUntil == null) return false;
823+
const untilMs = DateTime.toEpochMillis(thread.snoozedUntil);
824+
return (
825+
untilMs > nowMs &&
826+
!(thread.limitRecovery?.snooze === true && untilMs === Date.parse(thread.limitRecovery.resetAt))
827+
);
828+
}
829+
814830
function latestStableRun(
815831
projection: Pick<OrchestrationV2ThreadProjection, "runs">,
816832
): OrchestrationV2Run | null {
@@ -5252,8 +5268,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
52525268
projection.thread.settledOverride === "settled" ||
52535269
projection.thread.providerInstanceId !== run.providerInstanceId ||
52545270
projection.runtimeRequests.some((request) => request.status === "pending") ||
5255-
(projection.thread.snoozedUntil != null &&
5256-
DateTime.toEpochMillis(projection.thread.snoozedUntil) > DateTime.toEpochMillis(now))
5271+
snoozeHoldsLimitRecovery(projection.thread, DateTime.toEpochMillis(now))
52575272
) {
52585273
yield* emit(
52595274
events,
@@ -5268,9 +5283,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
52685283
recovery.requestId === command.usageLimitRecoveryRequestId &&
52695284
recovery.runId === command.usageLimitContinuationOfRunId &&
52705285
(projection.runtimeRequests.some((request) => request.status === "pending") ||
5271-
(projection.thread.snoozedUntil != null &&
5272-
DateTime.toEpochMillis(projection.thread.snoozedUntil) >
5273-
DateTime.toEpochMillis(now)))
5286+
snoozeHoldsLimitRecovery(projection.thread, DateTime.toEpochMillis(now)))
52745287
? { limitRecovery: { ...recovery, requestId: command.commandId } }
52755288
: {}),
52765289
},

‎apps/server/src/orchestration-v2/ProjectionStore.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3873,6 +3873,11 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
38733873
AND (
38743874
json_extract(t.payload_json, '$.snoozedUntil') IS NULL
38753875
OR julianday(json_extract(t.payload_json, '$.snoozedUntil')) <= julianday(${DateTime.formatIso(options.now)})
3876+
-- snoozeHoldsLimitRecovery: the recovery's own snooze yields to an early resume.
3877+
OR (
3878+
json_extract(t.payload_json, '$.limitRecovery.snooze') = 1
3879+
AND julianday(json_extract(t.payload_json, '$.snoozedUntil')) = julianday(json_extract(t.payload_json, '$.limitRecovery.resetAt'))
3880+
)
38763881
)
38773882
)
38783883
OR (

‎apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ export function limitRecoveryCommand(
7676
if (
7777
!recovery.autoResume ||
7878
(resetMs > nowMs && clearedAt === undefined) ||
79-
(thread.snoozedUntil != null && DateTime.toEpochMillis(thread.snoozedUntil) > nowMs)
79+
Orchestrator.snoozeHoldsLimitRecovery(thread, nowMs)
8080
)
8181
return null;
8282
// A separate id for early delivery keeps a rejected one from claiming the

‎apps/server/src/orchestration-v2/runtimeLayer.test.ts‎

Lines changed: 40 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -7388,15 +7388,17 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
73887388
}),
73897389
);
73907390

7391-
it.effect.each([true, false])(
7392-
"preserves early recovery with global auto-resume off when retry reports reset: %s",
7393-
(retryReportsReset) =>
7391+
it.effect.each(["reset", "no-reset", "snoozed"] as const)(
7392+
"preserves early recovery with global auto-resume off: %s",
7393+
(variant) =>
73947394
Effect.gen(function* () {
7395+
const retryReportsReset = variant === "reset";
7396+
const snoozed = variant === "snoozed";
73957397
const orchestrator = yield* Orchestrator.OrchestratorV2;
73967398
const events = yield* EventSink.EventSinkV2;
73977399
const projections = yield* ProjectionStore.ProjectionStoreV2;
7398-
const threadId = ThreadId.make(`recovery:early-reset:${retryReportsReset}`);
7399-
const projectId = ProjectId.make(`recovery:project:early-reset:${retryReportsReset}`);
7400+
const threadId = ThreadId.make(`recovery:early-reset:${variant}`);
7401+
const projectId = ProjectId.make(`recovery:project:early-reset:${variant}`);
74007402
const now = yield* DateTime.now;
74017403
yield* seedProject({
74027404
projectId,
@@ -7407,7 +7409,7 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
74077409
});
74087410
yield* orchestrator.dispatch({
74097411
type: "thread.create",
7410-
commandId: CommandId.make(`recovery:create:early-reset:${retryReportsReset}`),
7412+
commandId: CommandId.make(`recovery:create:early-reset:${variant}`),
74117413
threadId,
74127414
projectId,
74137415
title: "Limited thread",
@@ -7421,9 +7423,9 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
74217423
});
74227424
yield* orchestrator.dispatch({
74237425
type: "message.dispatch",
7424-
commandId: CommandId.make(`recovery:message:early-reset:${retryReportsReset}`),
7426+
commandId: CommandId.make(`recovery:message:early-reset:${variant}`),
74257427
threadId,
7426-
messageId: MessageId.make(`recovery:message:early-reset:${retryReportsReset}`),
7428+
messageId: MessageId.make(`recovery:message:early-reset:${variant}`),
74277429
text: "Work on this.",
74287430
attachments: [],
74297431
dispatchMode: { type: "defer_start" },
@@ -7435,19 +7437,19 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
74357437
yield* events.write({
74367438
events: [
74377439
{
7438-
id: EventId.make(`recovery:run:early-reset:${retryReportsReset}`),
7440+
id: EventId.make(`recovery:run:early-reset:${variant}`),
74397441
type: "run.updated",
74407442
threadId,
74417443
occurredAt: now,
74427444
payload: { ...run, status: "failed", completedAt: now },
74437445
},
74447446
{
7445-
id: EventId.make(`recovery:error:early-reset:${retryReportsReset}`),
7447+
id: EventId.make(`recovery:error:early-reset:${variant}`),
74467448
type: "turn-item.updated",
74477449
threadId,
74487450
occurredAt: now,
74497451
payload: {
7450-
id: TurnItemId.make(`recovery:error:early-reset:${retryReportsReset}`),
7452+
id: TurnItemId.make(`recovery:error:early-reset:${variant}`),
74517453
type: "error",
74527454
threadId,
74537455
runId: run.id,
@@ -7476,7 +7478,7 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
74767478
const shell = (yield* orchestrator.getShellSnapshot()).threads.find(
74777479
(thread) => thread.id === threadId,
74787480
)!;
7479-
const arm = limitRecoveryCommand(shell, true, DateTime.toEpochMillis(now));
7481+
const arm = limitRecoveryCommand(shell, true, DateTime.toEpochMillis(now), snoozed);
74807482
assert(arm?.type === "thread.metadata.update" && arm.limitRecovery != null);
74817483
yield* orchestrator.autoArmLimitRecovery({
74827484
commandId: arm.commandId,
@@ -7543,6 +7545,23 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
75437545
yield* sweep(reading(checkedNow, 100));
75447546
yield* sweep(reading(checkedNow, 0), alternateInstanceId);
75457547
assert.lengthOf((yield* orchestrator.getThreadProjection(threadId)).runs, 1);
7548+
if (snoozed) {
7549+
// A sidebar snooze holds the thread even when it ends at the reset.
7550+
yield* orchestrator.dispatch({
7551+
type: "thread.snooze",
7552+
commandId: CommandId.make(`recovery:user-snooze:early-reset:${variant}`),
7553+
threadId,
7554+
snoozedUntil: resetAt,
7555+
});
7556+
yield* sweep(reading(checkedNow, 0));
7557+
assert.lengthOf((yield* orchestrator.getThreadProjection(threadId)).runs, 1);
7558+
yield* orchestrator.dispatch({
7559+
type: "thread.metadata.update",
7560+
commandId: CommandId.make(`recovery:resnooze:early-reset:${variant}`),
7561+
threadId,
7562+
limitRecovery: { runId: run.id, resetAt, snooze: true },
7563+
});
7564+
}
75467565
const staleResume = limitRecoveryCommand(
75477566
armed!,
75487567
true,
@@ -7553,7 +7572,7 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
75537572
assert(staleResume?.type === "message.dispatch");
75547573
yield* orchestrator.dispatch({
75557574
type: "thread.metadata.update",
7556-
commandId: CommandId.make(`recovery:cancel:early-reset:${retryReportsReset}`),
7575+
commandId: CommandId.make(`recovery:cancel:early-reset:${variant}`),
75577576
threadId,
75587577
limitRecovery: { runId: run.id, resetAt, autoResume: false },
75597578
});
@@ -7562,7 +7581,7 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
75627581
assert.lengthOf((yield* orchestrator.getThreadProjection(threadId)).runs, 1);
75637582
yield* orchestrator.dispatch({
75647583
type: "thread.metadata.update",
7565-
commandId: CommandId.make(`recovery:rearm:early-reset:${retryReportsReset}`),
7584+
commandId: CommandId.make(`recovery:rearm:early-reset:${variant}`),
75667585
threadId,
75677586
limitRecovery: { runId: run.id, resetAt, autoResume: true },
75687587
});
@@ -7576,7 +7595,7 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
75767595
const nodeId = run.rootNodeId;
75777596
assert(nodeId !== null);
75787597
const request = {
7579-
id: RuntimeRequestId.make(`recovery:pending-race:${retryReportsReset}`),
7598+
id: RuntimeRequestId.make(`recovery:pending-race:${variant}`),
75807599
nodeId,
75817600
providerTurnId: null,
75827601
nativeRequestRef: null,
@@ -7589,7 +7608,7 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
75897608
yield* events.write({
75907609
events: [
75917610
{
7592-
id: EventId.make(`recovery:pending-race:${retryReportsReset}`),
7611+
id: EventId.make(`recovery:pending-race:${variant}`),
75937612
type: "runtime-request.updated",
75947613
threadId,
75957614
nodeId,
@@ -7603,7 +7622,7 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
76037622
yield* events.write({
76047623
events: [
76057624
{
7606-
id: EventId.make(`recovery:pending-race:resolved:${retryReportsReset}`),
7625+
id: EventId.make(`recovery:pending-race:resolved:${variant}`),
76077626
type: "runtime-request.updated",
76087627
threadId,
76097628
nodeId,
@@ -7616,6 +7635,7 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
76167635
const resumed = yield* orchestrator.getThreadProjection(threadId);
76177636
assert.lengthOf(resumed.runs, 2);
76187637
assert.equal(resumed.runs[1]?.status, "starting");
7638+
assert.isNull(resumed.thread.snoozedUntil);
76197639

76207640
// The reading can omit the limit that stopped the run, such as a
76217641
// per-model weekly limit. The early run then fails on the same limit.
@@ -7626,19 +7646,19 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
76267646
yield* events.write({
76277647
events: [
76287648
{
7629-
id: EventId.make(`recovery:run:early-reset:again:${retryReportsReset}`),
7649+
id: EventId.make(`recovery:run:early-reset:again:${variant}`),
76307650
type: "run.updated",
76317651
threadId,
76327652
occurredAt: checkedNow,
76337653
payload: { ...earlyRun, status: "failed", completedAt: checkedNow },
76347654
},
76357655
{
7636-
id: EventId.make(`recovery:error:early-reset:again:${retryReportsReset}`),
7656+
id: EventId.make(`recovery:error:early-reset:again:${variant}`),
76377657
type: "turn-item.updated",
76387658
threadId,
76397659
occurredAt: checkedNow,
76407660
payload: {
7641-
id: TurnItemId.make(`recovery:error:early-reset:again:${retryReportsReset}`),
7661+
id: TurnItemId.make(`recovery:error:early-reset:again:${variant}`),
76427662
type: "error",
76437663
threadId,
76447664
runId: earlyRun.id,

0 commit comments

Comments
 (0)