Skip to content

Commit 60f2246

Browse files
fix(server): stopped usage-limit threads no longer auto-resume (pingdotgg#21114)
Co-authored-by: Julius Marminge <[email protected]>
1 parent 4bf8b47 commit 60f2246

6 files changed

Lines changed: 196 additions & 8 deletions

File tree

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 71 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import {
2727
type ModelSelection,
2828
OrchestrationV2Command,
2929
type OrchestrationV2InternalCommand,
30+
type OrchestrationV2LimitRecoveryUpdate,
3031
type OrchestrationV2ServerCommand,
3132
type ThreadPullRequestLink,
3233
type ThreadPullRequestWatch,
@@ -312,6 +313,11 @@ export interface OrchestratorV2Shape {
312313
readonly dispatch: (
313314
command: OrchestrationV2ServerCommand,
314315
) => Effect.Effect<OrchestratorV2DispatchResult, OrchestratorV2Error>;
316+
readonly autoArmLimitRecovery: (input: {
317+
readonly commandId: CommandId;
318+
readonly threadId: ThreadId;
319+
readonly limitRecovery: OrchestrationV2LimitRecoveryUpdate;
320+
}) => Effect.Effect<void, OrchestratorV2Error>;
315321
readonly searchThreadStream: (
316322
input: OrchestrationV2SearchThreadInput,
317323
) => Stream.Stream<OrchestrationV2SearchThreadResult, OrchestratorV2Error>;
@@ -8978,7 +8984,49 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
89788984
readonly now: DateTime.Utc;
89798985
}) {
89808986
const emitEvent = emit(input.events, input.command);
8981-
const { thread, runs, subagents } = input.projection;
8987+
const { runs, subagents } = input.projection;
8988+
let { thread } = input.projection;
8989+
let recovery = thread.limitRecovery;
8990+
const latest = latestExecutedRun(runs);
8991+
if (latest?.status === "failed") {
8992+
const { turnItems } = yield* projectionStore
8993+
.getThreadRecords(thread.id, ["turnItems"], {
8994+
turnItemTypes: ["error"],
8995+
turnItemRunIds: [latest.id],
8996+
})
8997+
.pipe(mapDispatchError(input.command));
8998+
const failure = latestRootProviderFailure(latest, turnItems);
8999+
if (
9000+
failure?.class === "usage_limit" &&
9001+
failure.resetAt != null &&
9002+
Number.isFinite(Date.parse(failure.resetAt)) &&
9003+
(recovery?.runId !== latest.id || recovery.resetAt !== failure.resetAt)
9004+
) {
9005+
recovery = {
9006+
runId: latest.id,
9007+
resetAt: failure.resetAt,
9008+
autoResume: false,
9009+
snooze: false,
9010+
};
9011+
}
9012+
}
9013+
if (recovery && (recovery !== thread.limitRecovery || recovery.autoResume)) {
9014+
thread = {
9015+
...thread,
9016+
limitRecovery: {
9017+
...recovery,
9018+
autoResume: false,
9019+
requestId: input.command.commandId,
9020+
},
9021+
updatedAt: input.now,
9022+
};
9023+
yield* emitEvent({
9024+
type: "thread.metadata-updated",
9025+
threadId: thread.id,
9026+
occurredAt: input.now,
9027+
payload: thread,
9028+
});
9029+
}
89829030
const queuedRuns = runs.filter((run) => run.status === "queued");
89839031
const queuedMessages =
89849032
queuedRuns.length === 0
@@ -11760,6 +11808,26 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1176011808
awaitRestartContinuation: (run) =>
1176111809
effectOutbox.awaitSettled(`effect:restart-continuation:${run.id}`).pipe(Effect.ignore),
1176211810
dispatch: dispatchWithReceipt,
11811+
autoArmLimitRecovery: (input) =>
11812+
threadDispatch.withLock(
11813+
input.threadId,
11814+
Effect.gen(function* () {
11815+
const thread = yield* projectionStore
11816+
.getThread(input.threadId)
11817+
.pipe(
11818+
Effect.mapError(
11819+
(cause) => new OrchestratorProjectionError({ threadId: input.threadId, cause }),
11820+
),
11821+
);
11822+
// Stop or an explicit recovery choice can commit after the sweep reads its candidate.
11823+
if (
11824+
thread.limitRecovery?.runId === input.limitRecovery.runId &&
11825+
thread.limitRecovery.resetAt === input.limitRecovery.resetAt
11826+
)
11827+
return;
11828+
yield* dispatchWithReceiptEffect({ type: "thread.metadata.update", ...input });
11829+
}),
11830+
),
1176311831
searchThreadStream: (input) =>
1176411832
projectionStore
1176511833
.searchThreadStream(input)
@@ -11909,6 +11977,8 @@ const layerUnavailable: Layer.Layer<OrchestratorV2> = Layer.succeed(
1190911977
delegatedTaskResultPending: () => Effect.succeed(false),
1191011978
restartContinuationPending: () => Effect.succeed(false),
1191111979
awaitRestartContinuation: () => Effect.void,
11980+
autoArmLimitRecovery: (input) =>
11981+
Effect.fail(new OrchestratorProjectionError({ threadId: input.threadId })),
1191211982
dispatch: (command) =>
1191311983
Effect.fail(
1191411984
new OrchestratorDispatchError({

‎apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.ts‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import * as Layer from "effect/Layer";
55
import * as Scheduler from "../scheduling/Scheduler.ts";
66
import * as ServerSettings from "../serverSettings.ts";
77
import * as ProjectionStore from "./ProjectionStore.ts";
8+
import * as Orchestrator from "./Orchestrator.ts";
89
import * as ThreadManagement from "./ThreadManagementService.ts";
910

1011
export function limitRecoveryCommand(
@@ -73,6 +74,7 @@ export function limitRecoveryCommand(
7374
const makeSweep = Effect.gen(function* () {
7475
const projections = yield* ProjectionStore.ProjectionStoreV2;
7576
const threads = yield* ThreadManagement.ThreadManagementService;
77+
const orchestrator = yield* Orchestrator.OrchestratorV2;
7678
const settings = yield* ServerSettings.ServerSettingsService;
7779
return Effect.fn("UsageLimitRecoveryWorker.sweep")(function* () {
7880
const preferences = yield* settings.getSettings;
@@ -91,7 +93,17 @@ const makeSweep = Effect.gen(function* () {
9193
preferences.snoozeLimitedThreads,
9294
);
9395
if (command === null) continue;
94-
yield* threads.dispatch(command).pipe(
96+
yield* Effect.gen(function* () {
97+
if (command.type === "thread.metadata.update" && command.limitRecovery != null) {
98+
yield* orchestrator.autoArmLimitRecovery({
99+
commandId: command.commandId,
100+
threadId: command.threadId,
101+
limitRecovery: command.limitRecovery,
102+
});
103+
} else {
104+
yield* threads.dispatch(command);
105+
}
106+
}).pipe(
95107
Effect.catchCause((cause) =>
96108
Effect.logWarning("orchestration-v2.limit-recovery.dispatch-failed", {
97109
threadId: thread.id,

‎apps/server/src/orchestration-v2/runtimeLayer.test.ts‎

Lines changed: 101 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6198,6 +6198,9 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
61986198
"independent-patches",
61996199
"expired-snooze",
62006200
"wake",
6201+
"stop",
6202+
"stop-before-arm",
6203+
"stop-before-arm-mismatched-recovery",
62016204
] as const)("guards a scheduled usage-limit continuation against %s", (scenario) =>
62026205
Effect.gen(function* () {
62036206
const orchestrator = yield* Orchestrator.OrchestratorV2;
@@ -6354,8 +6357,68 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
63546357
].includes(scenario);
63556358
const autoResume = scenario !== "snooze-only" && scenario !== "wake";
63566359
const arm = limitRecoveryCommand(shell, autoResume, DateTime.toEpochMillis(now), snooze);
6357-
assert.isNotNull(arm);
6358-
yield* orchestrator.dispatch(arm!);
6360+
assert(arm?.type === "thread.metadata.update" && arm.limitRecovery != null);
6361+
const automaticArm = {
6362+
commandId: arm.commandId,
6363+
threadId,
6364+
limitRecovery: arm.limitRecovery,
6365+
};
6366+
const stopBeforeArm =
6367+
scenario === "stop-before-arm" || scenario === "stop-before-arm-mismatched-recovery";
6368+
if (stopBeforeArm) {
6369+
if (scenario === "stop-before-arm-mismatched-recovery") {
6370+
const current = yield* orchestrator.getThreadProjection(threadId);
6371+
yield* events.write({
6372+
events: [
6373+
{
6374+
id: EventId.make(`recovery:older-window:${scenario}`),
6375+
type: "thread.metadata-updated",
6376+
threadId,
6377+
occurredAt: now,
6378+
payload: {
6379+
...current.thread,
6380+
limitRecovery: {
6381+
runId: run.id,
6382+
resetAt: DateTime.formatIso(DateTime.add(now, { minutes: 2 })),
6383+
autoResume: true,
6384+
snooze: false,
6385+
},
6386+
},
6387+
},
6388+
],
6389+
});
6390+
}
6391+
yield* orchestrator.dispatch({
6392+
type: "thread.stop",
6393+
commandId: CommandId.make(`recovery:stop:${scenario}`),
6394+
threadId,
6395+
});
6396+
const projections = yield* ProjectionStore.ProjectionStoreV2;
6397+
assert.isUndefined(
6398+
(yield* projections.getLimitRecoveryCandidates({
6399+
now,
6400+
autoResume: true,
6401+
snooze: false,
6402+
})).find((row) => row.id === threadId),
6403+
);
6404+
const stoppedShell = (yield* orchestrator.getShellSnapshot()).threads.find(
6405+
(thread) => thread.id === threadId,
6406+
)!;
6407+
assert.isNull(limitRecoveryCommand(stoppedShell, true, DateTime.toEpochMillis(now)));
6408+
yield* orchestrator.autoArmLimitRecovery(automaticArm);
6409+
assert.deepEqual((yield* orchestrator.getThreadProjection(threadId)).thread.limitRecovery, {
6410+
runId: run.id,
6411+
resetAt,
6412+
autoResume: false,
6413+
snooze: false,
6414+
requestId: CommandId.make(`recovery:stop:${scenario}`),
6415+
});
6416+
}
6417+
const armCommand = stopBeforeArm
6418+
? { ...arm, commandId: CommandId.make(`recovery:explicit-rearm:${scenario}`) }
6419+
: arm;
6420+
if (stopBeforeArm) yield* orchestrator.dispatch(armCommand);
6421+
else yield* orchestrator.autoArmLimitRecovery(automaticArm);
63596422
let armedShell = (yield* orchestrator.getShellSnapshot()).threads.find(
63606423
(thread) => thread.id === threadId,
63616424
)!;
@@ -6364,7 +6427,7 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
63646427
resetAt,
63656428
autoResume,
63666429
snooze,
6367-
requestId: arm!.commandId,
6430+
requestId: armCommand.commandId,
63686431
});
63696432
if (snooze)
63706433
assert.equal(DateTime.toEpochMillis(armedShell.snoozedUntil!), Date.parse(resetAt));
@@ -6568,6 +6631,17 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
65686631
commandId: CommandId.make(`recovery:settle:${scenario}`),
65696632
threadId,
65706633
});
6634+
if (scenario === "stop") {
6635+
yield* orchestrator.dispatch({
6636+
type: "thread.stop",
6637+
commandId: CommandId.make(`recovery:stop:${scenario}`),
6638+
threadId,
6639+
});
6640+
const stoppedShell = (yield* orchestrator.getShellSnapshot()).threads.find(
6641+
(thread) => thread.id === threadId,
6642+
)!;
6643+
assert.deepInclude(stoppedShell.limitRecovery, { runId: run.id, autoResume: false });
6644+
}
65716645
if (scenario === "replacement") {
65726646
const current = yield* orchestrator.getThreadProjection(threadId);
65736647
const error = current.turnItems.find((item) => item.type === "error")!;
@@ -6626,6 +6700,7 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
66266700
after.runs,
66276701
before.runs.length +
66286702
(scenario === "resume" ||
6703+
stopBeforeArm ||
66296704
scenario === "queued-resume" ||
66306705
scenario === "snooze-resume" ||
66316706
scenario === "wake-preserve-resume" ||
@@ -6638,6 +6713,7 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
66386713
after.messages,
66396714
before.messages.length +
66406715
(scenario === "resume" ||
6716+
stopBeforeArm ||
66416717
scenario === "queued-resume" ||
66426718
scenario === "snooze-resume" ||
66436719
scenario === "wake-preserve-resume" ||
@@ -6646,6 +6722,28 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
66466722
? 1
66476723
: 0),
66486724
);
6725+
if (scenario === "stop") {
6726+
assert.deepInclude(after.thread.limitRecovery, { runId: run.id, autoResume: false });
6727+
const projections = yield* ProjectionStore.ProjectionStoreV2;
6728+
const sweep = { now: yield* DateTime.now, autoResume: true, snooze: false };
6729+
assert.isUndefined(
6730+
(yield* projections.getLimitRecoveryCandidates(sweep)).find((row) => row.id === threadId),
6731+
);
6732+
yield* orchestrator.dispatch({
6733+
type: "thread.metadata.update",
6734+
commandId: CommandId.make("recovery:rearm-after-stop"),
6735+
threadId,
6736+
limitRecovery: { runId: run.id, resetAt, autoResume: true },
6737+
});
6738+
const rearmed = (yield* projections.getLimitRecoveryCandidates(sweep)).find(
6739+
(row) => row.id === threadId,
6740+
)!;
6741+
assert.isDefined(rearmed);
6742+
const freshResume = limitRecoveryCommand(rearmed, true, DateTime.toEpochMillis(sweep.now));
6743+
assert.isNotNull(freshResume);
6744+
yield* orchestrator.dispatch(freshResume!);
6745+
assert.lengthOf((yield* orchestrator.getThreadProjection(threadId)).runs, 2);
6746+
}
66496747
if (scenario === "queued-resume") {
66506748
const held = after.runs[1]!;
66516749
assert.equal(held.status, "queued");

‎apps/server/src/orchestration-v2/runtimeLayer.ts‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -368,7 +368,13 @@ export const layerProduction = Layer.mergeAll(
368368
layerScheduledTaskProvided,
369369
layerSecretRequestsProvided,
370370
UsageLimitRecoveryWorker.layer.pipe(
371-
Layer.provide(Layer.mergeAll(ProjectionStore.layer, layerThreadManagementProvided)),
371+
Layer.provide(
372+
Layer.mergeAll(
373+
ProjectionStore.layer,
374+
layerThreadManagementProvided,
375+
layerOrchestratorProvided,
376+
),
377+
),
372378
),
373379
layerProviderContinuationWorkerProvided,
374380
layerAgentSessionImporterProvided,

‎apps/server/src/scheduling/Scheduler.integration.test.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import * as SqlClient from "effect/sql/SqlClient";
1919
import * as TestClock from "effect/testing/TestClock";
2020

2121
import * as ProjectionStore from "../orchestration-v2/ProjectionStore.ts";
22+
import * as Orchestrator from "../orchestration-v2/Orchestrator.ts";
2223
import * as ThreadLaunchService from "../orchestration-v2/ThreadLaunchService.ts";
2324
import * as ThreadManagementService from "../orchestration-v2/ThreadManagementService.ts";
2425
import * as UsageLimitRecoveryWorker from "../orchestration-v2/UsageLimitRecoveryWorker.ts";
@@ -89,6 +90,7 @@ it.effect.each(["on time", "after restart"])(
8990
const receipts = yield* Queue.unbounded<"task" | "retry">();
9091
const layerDependencies = Layer.mergeAll(
9192
NodeCrypto.layer,
93+
Layer.mock(Orchestrator.OrchestratorV2)({}),
9294
Layer.mock(ThreadLaunchService.ThreadLaunchService)({
9395
launch: () =>
9496
Queue.offer(receipts, "task").pipe(

‎docs/user/thread-sidebar.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -206,8 +206,8 @@ continuation. You can cancel it from the thread. Enable **Auto-resume limited
206206
threads** in **Settings → General** on web and desktop, or **Settings → Thread
207207
behavior** on mobile, to schedule limit stops by default.
208208
The environment must be running when the reset arrives; it resumes overdue
209-
continuations after a restart. Sending a new message, archiving, or settling the
210-
thread prevents a pending continuation from starting.
209+
continuations after a restart. Stopping the thread, sending a new message,
210+
archiving, or settling it prevents a pending continuation from starting.
211211

212212
Choose **Snooze until reset** to hide the thread until its allowance returns.
213213
Snooze and auto-resume are independent: snooze alone wakes the thread without

0 commit comments

Comments
 (0)