Skip to content

Commit 78d8f97

Browse files
committed
fix(server): scope recovery waits and retain reset controls
1 parent 24846a0 commit 78d8f97

4 files changed

Lines changed: 59 additions & 14 deletions

File tree

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 2 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -804,18 +804,11 @@ function usageLimitHoldsQueue(
804804
);
805805
}
806806

807-
// A run that hit the limit again before the previous reset outlasted what the
808-
// usage reading showed, so its recovery waits for the reset instead of resuming
809-
// early on that reading once more.
810807
function limitRecoveryWaitsForReset(
811808
previous: OrchestrationV2LimitRecovery | null | undefined,
812809
runId: RunId,
813-
now: DateTime.Utc,
814810
): boolean {
815-
if (previous == null) return false;
816-
return previous.runId === runId
817-
? previous.waitForReset === true
818-
: Date.parse(previous.resetAt) > DateTime.toEpochMillis(now);
811+
return previous?.runId === runId && previous.waitForReset === true;
819812
}
820813

821814
function latestStableRun(
@@ -3509,7 +3502,6 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
35093502
...(limitRecoveryWaitsForReset(
35103503
thread.limitRecovery,
35113504
command.limitRecovery.runId,
3512-
now,
35133505
)
35143506
? { waitForReset: true }
35153507
: {}),
@@ -10471,9 +10463,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1047110463
resetAt: failure.resetAt,
1047210464
autoResume: false,
1047310465
snooze: false,
10474-
...(limitRecoveryWaitsForReset(recovery, latest.id, input.now)
10475-
? { waitForReset: true }
10476-
: {}),
10466+
...(limitRecoveryWaitsForReset(recovery, latest.id) ? { waitForReset: true } : {}),
1047710467
};
1047810468
}
1047910469
}

‎apps/server/src/orchestration-v2/ProjectionStore.test.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1047,6 +1047,12 @@ it.effect("SQL limit recovery loads armed threads before their reset", () =>
10471047
payload: { ...failure, failure: { ...failure.failure!, resetAt: null } },
10481048
});
10491049
assert.equal((yield* candidate)?.usageLimitResetAt, retry.resetAt);
1050+
assert.equal((yield* store.getThreadShell(threadId))?.usageLimitResetAt, retry.resetAt);
1051+
assert.equal(
1052+
ProjectionStore.threadShellFromProjection(yield* store.getThreadProjection(threadId))
1053+
.usageLimitResetAt,
1054+
retry.resetAt,
1055+
);
10501056
yield* updateThread("no-reset-unarmed", {});
10511057
assert.isUndefined(yield* candidate);
10521058
yield* updateThread("retry-cancelled", { limitRecovery: { ...retry, autoResume: false } });

‎apps/server/src/orchestration-v2/ProjectionStore.ts‎

Lines changed: 26 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1403,6 +1403,26 @@ const waitsOnUser = (item: OrchestrationV2TurnItem, liveRunIds: ReadonlySet<RunI
14031403
item.type === "link_request" ||
14041404
(item.type === "secret_request" && item.runId !== null && liveRunIds.has(item.runId));
14051405

1406+
function shellErrorSummary(
1407+
failure: Parameters<typeof threadErrorSummary>[0],
1408+
sessionError: string | null,
1409+
thread: OrchestrationV2AppThread,
1410+
runId: RunId | null,
1411+
) {
1412+
const summary = threadErrorSummary(failure, sessionError);
1413+
const recovery = thread.limitRecovery;
1414+
return {
1415+
...summary,
1416+
usageLimitResetAt:
1417+
summary.usageLimitResetAt ??
1418+
(summary.lastErrorClass === "usage_limit" &&
1419+
recovery?.runId === runId &&
1420+
recovery.waitForReset === true
1421+
? recovery.resetAt
1422+
: null),
1423+
};
1424+
}
1425+
14061426
export function threadShellFromProjection(
14071427
projection: OrchestrationV2ThreadProjection,
14081428
): OrchestrationV2ThreadShell {
@@ -1513,9 +1533,11 @@ export function threadShellFromProjection(
15131533
activityRunStartedAt:
15141534
activityRun === null ? null : orchestrationV2RunWorkStartedAt(activityRun),
15151535
status: latestRun?.status ?? "idle",
1516-
...threadErrorSummary(
1536+
...shellErrorSummary(
15171537
latestRootProviderFailure(latestRun, projection.turnItems),
15181538
providerSession?.lastError ?? null,
1539+
projection.thread,
1540+
latestRun?.id ?? null,
15191541
),
15201542
pendingRuntimeRequest,
15211543
pendingGuestMessageCount: pendingGuestMessageCount(projection.thread.guestShare),
@@ -6476,9 +6498,11 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
64766498
row.activity_run_status === "waiting"
64776499
? row.activity_run_status
64786500
: null,
6479-
...threadErrorSummary(
6501+
...shellErrorSummary(
64806502
terminalFailureItem?.type === "error" ? terminalFailureItem.failure : null,
64816503
row.last_error,
6504+
thread,
6505+
latestRunId,
64826506
),
64836507
pendingRuntimeRequest,
64846508
latestUserMessageAt:

‎apps/server/src/orchestration-v2/runtimeLayer.test.ts‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7621,6 +7621,7 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
76217621
"stop",
76227622
"stop-before-arm",
76237623
"stop-before-arm-mismatched-recovery",
7624+
"unrelated-recovery",
76247625
] as const)("guards a scheduled usage-limit continuation against %s", (scenario) =>
76257626
Effect.gen(function* () {
76267627
const orchestrator = yield* Orchestrator.OrchestratorV2;
@@ -7776,6 +7777,28 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
77767777
threadId,
77777778
limitRecovery: arm.limitRecovery,
77787779
};
7780+
if (scenario === "unrelated-recovery") {
7781+
const current = yield* orchestrator.getThreadProjection(threadId);
7782+
yield* events.write({
7783+
events: [
7784+
{
7785+
id: EventId.make("recovery:unrelated-window"),
7786+
type: "thread.metadata-updated",
7787+
threadId,
7788+
occurredAt: now,
7789+
payload: {
7790+
...current.thread,
7791+
limitRecovery: {
7792+
runId: RunId.make("recovery:old-provider-run"),
7793+
resetAt: DateTime.formatIso(DateTime.add(now, { hours: 4 })),
7794+
autoResume: true,
7795+
waitForReset: true,
7796+
},
7797+
},
7798+
},
7799+
],
7800+
});
7801+
}
77797802
const stopBeforeArm =
77807803
scenario === "stop-before-arm" || scenario === "stop-before-arm-mismatched-recovery";
77817804
if (stopBeforeArm) {
@@ -8089,6 +8112,7 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
80898112
after.runs,
80908113
before.runs.length +
80918114
(scenario === "resume" ||
8115+
scenario === "unrelated-recovery" ||
80928116
stopBeforeArm ||
80938117
scenario === "queued-resume" ||
80948118
scenario === "independent-patches" ||
@@ -8100,6 +8124,7 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
81008124
after.messages,
81018125
before.messages.length +
81028126
(scenario === "resume" ||
8127+
scenario === "unrelated-recovery" ||
81038128
stopBeforeArm ||
81048129
scenario === "queued-resume" ||
81058130
scenario === "independent-patches" ||

0 commit comments

Comments
 (0)