Skip to content

Commit 0a412bb

Browse files
ScottN-PVclaude
andcommitted
fix(server): keep threads armed to auto-resume out of inactivity auto-settle
A thread that hit a usage limit with auto-resume armed looked like any idle thread to inactivity auto-settle, so a reset further away than the settle window let the sweep settle it first, and settling drops the resume. An armed resume for the latest run now parks the thread like a snooze until its reset, and the reset restarts the inactivity window. Settlement candidates carry limitRecovery so the sweep can see it. Closes #18928 Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 09c5e7a commit 0a412bb

4 files changed

Lines changed: 88 additions & 4 deletions

File tree

‎apps/server/src/orchestration-v2/ProjectionSettlement.test.ts‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -212,6 +212,16 @@ it.effect.each([
212212
snoozedAt: now,
213213
}),
214214
);
215+
yield* createRun(
216+
yield* createThread("limit-armed", {
217+
limitRecovery: {
218+
runId: RunId.make("run:thread:settlement:limit-armed:1"),
219+
resetAt: DateTime.formatIso(DateTime.add(now, { days: 1 })),
220+
autoResume: true,
221+
},
222+
}),
223+
"failed",
224+
);
215225
const woke = yield* createThread("woke", {
216226
snoozedUntil: DateTime.add(now, { days: 1 }),
217227
snoozedAt: DateTime.subtract(old, { days: 1 }),
@@ -270,6 +280,7 @@ it.effect.each([
270280
for (const candidate of candidates) {
271281
const expected = shell.threads.find((thread) => thread.id === candidate.id)!;
272282
assert.deepEqual(candidate.pendingBackgroundTasks, expected.pendingBackgroundTasks);
283+
assert.deepEqual(candidate.limitRecovery, expected.limitRecovery);
273284
const settings = {
274285
pullRequest: null,
275286
nowMs: DateTime.toEpochMillis(now),

‎apps/server/src/orchestration-v2/ProjectionStore.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -201,6 +201,7 @@ export type ProjectionSettlementCandidate = Pick<
201201
| "activityRunStatus"
202202
| "pendingRuntimeRequest"
203203
| "pendingBackgroundTasks"
204+
| "limitRecovery"
204205
> &
205206
Pick<OrchestrationV2AppThread, "lastSnoozeWakeAt"> & {
206207
readonly latestUserAuthoredMessageAt: DateTime.Utc | null;
@@ -5596,6 +5597,7 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
55965597
autoSettleDisabledAt: thread.autoSettleDisabledAt ?? null,
55975598
snoozedUntil: thread.snoozedUntil ?? null,
55985599
snoozedAt: thread.snoozedAt ?? null,
5600+
limitRecovery: thread.limitRecovery ?? null,
55995601
status,
56005602
latestRunId,
56015603
latestRunRequestedAt:

‎apps/server/src/orchestration-v2/ThreadSettlementService.test.ts‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import {
77
EventId,
88
ProviderInstanceId,
99
ProviderSessionId,
10+
RunId,
1011
ThreadId,
1112
type OrchestrationProjectShell,
1213
type OrchestrationV2AppThread,
@@ -206,6 +207,29 @@ describe("isAutoSettlementCandidate", () => {
206207
),
207208
).toBe(true);
208209
});
210+
211+
it("keeps a thread armed to auto-resume parked until its usage limit resets", () => {
212+
const recovery = {
213+
runId: RunId.make("run-1"),
214+
resetAt: DateTime.formatIso(at(3 * DAY_MS)),
215+
autoResume: true,
216+
};
217+
const armed = shell({
218+
status: "failed",
219+
latestRunId: RunId.make("run-1"),
220+
latestRunCompletedAt: at(-4 * DAY_MS),
221+
limitRecovery: recovery,
222+
});
223+
expect(ThreadSettlementService.isAutoSettlementCandidate(armed, NOW_MS)).toBe(false);
224+
for (const thread of [
225+
shell({ ...armed, limitRecovery: { ...recovery, autoResume: false } }),
226+
shell({ ...armed, limitRecovery: { ...recovery, runId: RunId.make("run-0") } }),
227+
shell({ ...armed, latestRunId: null }),
228+
shell({ ...armed, limitRecovery: { ...recovery, resetAt: DateTime.formatIso(at(-1_000)) } }),
229+
]) {
230+
expect(ThreadSettlementService.isAutoSettlementCandidate(thread, NOW_MS)).toBe(true);
231+
}
232+
});
209233
});
210234

211235
describe("threadHasQueuedTurnStart", () => {
@@ -322,6 +346,33 @@ describe("resolveAutoSettlementAt", () => {
322346
).toEqual(at(-1_000));
323347
});
324348

349+
it("waits out an armed auto-resume and restarts inactivity at its reset", () => {
350+
const thread = shell({
351+
status: "failed",
352+
latestRunId: RunId.make("run-1"),
353+
latestRunCompletedAt: at(-4 * DAY_MS),
354+
limitRecovery: {
355+
runId: RunId.make("run-1"),
356+
resetAt: DateTime.formatIso(at(DAY_MS)),
357+
autoResume: true,
358+
},
359+
});
360+
const input = {
361+
thread,
362+
pullRequest: null,
363+
nowMs: NOW_MS,
364+
autoSettleAfterDays: 1,
365+
autoSettleOnMerge: false,
366+
};
367+
expect(ThreadSettlementService.resolveAutoSettlementAt(input)).toBeNull();
368+
expect(
369+
ThreadSettlementService.resolveAutoSettlementAt({ ...input, nowMs: NOW_MS + DAY_MS }),
370+
).toBeNull();
371+
expect(
372+
ThreadSettlementService.resolveAutoSettlementAt({ ...input, nowMs: NOW_MS + 2 * DAY_MS + 1 }),
373+
).toEqual(at(DAY_MS));
374+
});
375+
325376
it("settles on merge only after the user's last action and preserves the activity time", () => {
326377
const thread = shell({
327378
latestUserMessageAt: at(-2 * 60 * 60 * 1_000),

‎apps/server/src/orchestration-v2/ThreadSettlementService.ts‎

Lines changed: 24 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,18 @@ function pullRequestSettles(
128128
return pullRequestAtMs >= userAnchorMs;
129129
}
130130

131+
/** When an auto-resume armed for the thread's latest run will fire, or null. */
132+
function armedLimitResetAtMs(
133+
thread: Pick<ProjectionStore.ProjectionSettlementCandidate, "limitRecovery" | "latestRunId">,
134+
): number | null {
135+
const recovery = thread.limitRecovery;
136+
if (recovery == null || !recovery.autoResume || recovery.runId !== thread.latestRunId) {
137+
return null;
138+
}
139+
const resetAtMs = Date.parse(recovery.resetAt);
140+
return Number.isFinite(resetAtMs) ? resetAtMs : null;
141+
}
142+
131143
export function isAutoSettlementCandidate(
132144
thread: Omit<ProjectionStore.ProjectionSettlementCandidate, "latestUserAuthoredMessageAt">,
133145
nowMs: number,
@@ -138,6 +150,10 @@ export function isAutoSettlementCandidate(
138150
if (thread.activityRunStatus != null) return false;
139151
if (backgroundWorkHoldsCompletion(thread.pendingBackgroundTasks ?? [])) return false;
140152
if (threadHasQueuedTurnStart(thread, nowMs)) return false;
153+
// An armed auto-resume is a parked wake, like a snooze. Settling the thread
154+
// before its reset would drop the resume.
155+
const limitResetAtMs = armedLimitResetAtMs(thread);
156+
if (limitResetAtMs !== null && limitResetAtMs > nowMs) return false;
141157
const snoozedUntilMs = toMillis(thread.snoozedUntil);
142158
if (snoozedUntilMs === null || snoozedUntilMs <= nowMs) return true;
143159
const snoozedAtMs = toMillis(thread.snoozedAt);
@@ -222,11 +238,15 @@ export function resolveAutoSettlementAt(input: {
222238
pullRequest !== null || thread.linkedPullRequest != null || thread.branchPullRequest != null;
223239
if (input.autoSettleKeepPullRequests === true && hasPullRequest) return null;
224240
if (input.autoSettleAfterDays === null || activityAtMs === null) return null;
225-
const snoozedUntilMs = toMillis(thread.snoozedUntil);
226-
const timerWakeAtMs =
227-
snoozedUntilMs !== null && snoozedUntilMs <= input.nowMs ? snoozedUntilMs : null;
241+
const passedWakeAtMs = (wakeAtMs: number | null) =>
242+
wakeAtMs !== null && wakeAtMs <= input.nowMs ? wakeAtMs : null;
228243
const inactivityAtMs =
229-
latestMillis([activityAtMs, toMillis(thread.lastSnoozeWakeAt), timerWakeAtMs]) ?? activityAtMs;
244+
latestMillis([
245+
activityAtMs,
246+
toMillis(thread.lastSnoozeWakeAt),
247+
passedWakeAtMs(toMillis(thread.snoozedUntil)),
248+
passedWakeAtMs(armedLimitResetAtMs(thread)),
249+
]) ?? activityAtMs;
230250
return inactivityAtMs < input.nowMs - input.autoSettleAfterDays * DAY_MS
231251
? DateTime.makeUnsafe(inactivityAtMs)
232252
: null;

0 commit comments

Comments
 (0)