Skip to content

Commit df9fdff

Browse files
committed
fix: close automatic resume terminal gaps
1 parent 5e90188 commit df9fdff

7 files changed

Lines changed: 94 additions & 10 deletions

File tree

‎apps/server/src/orchestration/Layers/ProviderCommandReactor.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1800,7 +1800,10 @@ const make = Effect.gen(function* () {
18001800
const pendingUsageLimitResumes = yield* projectionSnapshotQuery.getCommandReadModel().pipe(
18011801
Effect.map((readModel) =>
18021802
readModel.threads.flatMap((thread) =>
1803-
thread.deletedAt !== null || thread.archivedAt !== null || thread.usageLimitResume == null
1803+
thread.deletedAt !== null ||
1804+
thread.archivedAt !== null ||
1805+
thread.settledOverride === "settled" ||
1806+
thread.usageLimitResume == null
18041807
? []
18051808
: [
18061809
{

‎apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.test.ts‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2819,6 +2819,25 @@ describe("ProviderRuntimeIngestion", () => {
28192819
nextAttemptAt: "2099-01-01T01:00:02.000Z",
28202820
attempt: 1,
28212821
});
2822+
2823+
harness.emit({
2824+
type: "turn.completed",
2825+
eventId: asEventId("evt-runtime-usage-limit-failed"),
2826+
provider: ProviderDriverKind.make("codex"),
2827+
createdAt: attemptedAt,
2828+
threadId: asThreadId("thread-1"),
2829+
turnId: asTurnId("turn-usage-limit"),
2830+
payload: { state: "failed" },
2831+
});
2832+
await harness.drain();
2833+
2834+
const afterCompletion = (await harness.readModel()).threads.find(
2835+
(entry) => entry.id === ThreadId.make("thread-1"),
2836+
);
2837+
expect(afterCompletion?.usageLimitResume).toEqual({
2838+
nextAttemptAt: "2099-01-01T01:00:02.000Z",
2839+
attempt: 1,
2840+
});
28222841
});
28232842

28242843
it("clears automatic resume after a successful provider turn", async () => {
@@ -2899,6 +2918,43 @@ describe("ProviderRuntimeIngestion", () => {
28992918
expect(thread.usageLimitResume).toBeNull();
29002919
});
29012920

2921+
it("clears an in-flight automatic resume after a failed turn without a runtime error", async () => {
2922+
const harness = await createHarness();
2923+
const attemptedAt = "2099-01-01T00:00:00.000Z";
2924+
2925+
await harness.markUsageLimited();
2926+
await harness.dispatch({
2927+
type: "thread.usage-limit-resume.schedule",
2928+
commandId: CommandId.make("cmd-failed-resume-schedule"),
2929+
threadId: ThreadId.make("thread-1"),
2930+
resumeAt: attemptedAt,
2931+
});
2932+
await harness.dispatch({
2933+
type: "thread.usage-limit-resume.attempt",
2934+
commandId: CommandId.make("cmd-failed-resume-attempt"),
2935+
threadId: ThreadId.make("thread-1"),
2936+
expectedAttemptAt: attemptedAt,
2937+
createdAt: attemptedAt,
2938+
});
2939+
2940+
harness.emit({
2941+
type: "turn.completed",
2942+
eventId: asEventId("evt-failed-auto-resume"),
2943+
provider: ProviderDriverKind.make("codex"),
2944+
createdAt: attemptedAt,
2945+
threadId: asThreadId("thread-1"),
2946+
turnId: asTurnId("turn-failed-auto-resume"),
2947+
payload: { state: "failed" },
2948+
});
2949+
2950+
const thread = await waitForThread(
2951+
harness.readModel,
2952+
(entry) => entry.usageLimitResume === null,
2953+
);
2954+
expect(thread.usageLimitResume).toBeNull();
2955+
expect(thread.session?.status).toBe("error");
2956+
});
2957+
29022958
it("keeps automatic resume active when a superseded turn completes", async () => {
29032959
const harness = await createHarness();
29042960
const attemptedAt = "2099-01-01T00:00:00.000Z";

‎apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1954,14 +1954,16 @@ const make = Effect.gen(function* () {
19541954
if (
19551955
shouldApplyThreadLifecycle &&
19561956
event.type === "turn.completed" &&
1957-
normalizeRuntimeTurnState(event.payload.state) !== "failed" &&
19581957
thread.usageLimitResume != null
19591958
) {
1960-
yield* orchestrationEngine.dispatch({
1961-
type: "thread.usage-limit-resume.cancel",
1962-
commandId: yield* providerCommandId(event, "usage-limit-resume-completed"),
1963-
threadId: thread.id,
1964-
});
1959+
const completedState = normalizeRuntimeTurnState(event.payload.state);
1960+
if (completedState !== "failed" || thread.usageLimitResume.nextAttemptAt === null) {
1961+
yield* orchestrationEngine.dispatch({
1962+
type: "thread.usage-limit-resume.cancel",
1963+
commandId: yield* providerCommandId(event, "usage-limit-resume-completed"),
1964+
threadId: thread.id,
1965+
});
1966+
}
19651967
}
19661968

19671969
if (event.type === "thread.metadata.updated" && event.payload.name) {

‎apps/server/src/orchestration/decider.ts‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -850,7 +850,11 @@ export const decideOrchestrationCommand = Effect.fn("decideOrchestrationCommand"
850850
command,
851851
threadId: command.threadId,
852852
});
853-
if (thread.deletedAt !== null || thread.archivedAt !== null) {
853+
if (
854+
thread.deletedAt !== null ||
855+
thread.archivedAt !== null ||
856+
thread.settledOverride === "settled"
857+
) {
854858
return {
855859
...(yield* withEventBase({
856860
aggregateKind: "thread",

‎apps/server/src/orchestration/decider.usageLimitResume.test.ts‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -300,6 +300,21 @@ it.layer(NodeServices.layer)("usage-limit resume decider", (it) => {
300300
const attemptedEvents = Array.isArray(attempted) ? attempted : [attempted];
301301
expect(attemptedEvents).toHaveLength(1);
302302
expect(attemptedEvents[0]?.type).toBe("thread.usage-limit-resume-cancelled");
303+
304+
const retried = yield* decideOrchestrationCommand({
305+
command: {
306+
type: "thread.usage-limit-resume.retry",
307+
commandId: CommandId.make("cmd-settled-retry"),
308+
threadId: ThreadId.make("thread-1"),
309+
resumeAt: SECOND_ATTEMPT,
310+
attempt: 0,
311+
createdAt: FIRST_ATTEMPT,
312+
},
313+
readModel: makeReadModel({ nextAttemptAt: null, attempt: 0 }, "settled"),
314+
});
315+
const retriedEvents = Array.isArray(retried) ? retried : [retried];
316+
expect(retriedEvents).toHaveLength(1);
317+
expect(retriedEvents[0]?.type).toBe("thread.usage-limit-resume-cancelled");
303318
}),
304319
);
305320

‎packages/client-runtime/src/state/usageLimitResume.test.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,10 @@ describe("initialUsageLimitResumeAt", () => {
66
it("uses a future provider reset timestamp", () => {
77
expect(
88
initialUsageLimitResumeAt("2026-08-28T19:10:00.000Z", Date.parse("2026-08-28T18:00:00Z")),
9-
).toBe("2026-08-28T19:10:00.000Z");
9+
).toBe("2026-08-28T19:10:02.000Z");
10+
expect(
11+
initialUsageLimitResumeAt("2026-08-28T18:00:00.500Z", Date.parse("2026-08-28T18:00:00Z")),
12+
).toBe("2026-08-28T18:00:02.500Z");
1013
});
1114

1215
it("falls back to five minutes when the provider has no future reset", () => {

‎packages/client-runtime/src/state/usageLimitResume.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import type { IsoDateTime } from "@t3tools/contracts";
22
import * as DateTime from "effect/DateTime";
33

44
const FALLBACK_RETRY_DELAY_MS = 5 * 60 * 1_000;
5+
const PROVIDER_RETRY_CUSHION_MS = 2_000;
56

67
export function initialUsageLimitResumeAt(
78
providerRetryAt: IsoDateTime | undefined,
@@ -11,7 +12,7 @@ export function initialUsageLimitResumeAt(
1112
providerRetryAt === undefined ? Number.NaN : Date.parse(providerRetryAt);
1213
const resumeAtMs =
1314
Number.isFinite(providerRetryAtMs) && providerRetryAtMs > nowMs
14-
? providerRetryAtMs
15+
? providerRetryAtMs + PROVIDER_RETRY_CUSHION_MS
1516
: nowMs + FALLBACK_RETRY_DELAY_MS;
1617
return DateTime.formatIso(DateTime.makeUnsafe(resumeAtMs)) as IsoDateTime;
1718
}

0 commit comments

Comments
 (0)