Skip to content

Commit e2afc1f

Browse files
feat: recover threads interrupted by provider usage limits (#101)
When Codex or Claude ends a turn because the account's usage allowance ran out, the thread now shows a recovery banner instead of a plain error. It can resume the same session now, arm an automatic resume for shortly after the reported reset (kept by the server across restarts, cancellable), or fork the chat into a new tab on another account or model. Adapters mark the stop with `usageLimit` on `runtime.error`; the `thread.usage-limit.resume` command records arming as thread activities and starts the continue turn; a reactor holds the timers. Co-authored-by: Claude Opus 5.5 (1M context) <[email protected]>
1 parent 6ea52f7 commit e2afc1f

31 files changed

Lines changed: 1471 additions & 31 deletions

‎apps/mobile/src/features/threads/ThreadComposer.tsx‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import { ChatGptUsageLimitNotice } from "./ChatGptUsageLimitNotice";
2+
import { UsageLimitRecoveryNotice } from "./UsageLimitRecoveryNotice";
23
import type { ComposerTextPaste } from "../../native/T3ComposerEditor.types";
34
import { useAppearancePreferences } from "../settings/appearance/AppearancePreferencesProvider";
45
import { useAtomValue } from "@effect/atom-react";
@@ -647,6 +648,10 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer
647648
environmentId={props.environmentId}
648649
thread={props.selectedThread}
649650
/>
651+
<UsageLimitRecoveryNotice
652+
environmentId={props.environmentId}
653+
thread={props.selectedThread}
654+
/>
650655
{!voiceInput.isBusy &&
651656
composerMenu.trigger &&
652657
(composerMenu.items.length > 0 || composerMenu.trigger.kind === "pull-request") ? (
Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
import type { EnvironmentId, OrchestrationThreadShell } from "@t3tools/contracts";
2+
import { deriveUsageLimitRecovery, usageLimitResumeAt } from "@t3tools/shared/usageLimitRecovery";
3+
import * as Cause from "effect/Cause";
4+
import * as Option from "effect/Option";
5+
import { useState } from "react";
6+
import { Alert, Pressable, View } from "react-native";
7+
8+
import { AppText as Text } from "../../components/AppText";
9+
import { useAtomValue } from "@effect/atom-react";
10+
import { environmentServerConfigsAtom } from "../../state/server";
11+
import { threadEnvironment } from "../../state/threads";
12+
import { useAtomCommand } from "../../state/use-atom-command";
13+
import { useThreadDetail } from "../../state/use-thread-detail";
14+
15+
const RESET_TIME_FORMATTER = new Intl.DateTimeFormat(undefined, {
16+
weekday: "short",
17+
hour: "numeric",
18+
minute: "2-digit",
19+
});
20+
21+
/**
22+
* Resume actions for a turn stopped on a usage limit. Continuing on another
23+
* account needs a chat fork, which mobile does not have; that stays on web and
24+
* desktop.
25+
*/
26+
export function UsageLimitRecoveryNotice({
27+
environmentId,
28+
thread,
29+
}: {
30+
environmentId: EnvironmentId;
31+
thread: OrchestrationThreadShell;
32+
}) {
33+
const state = useThreadDetail({ environmentId, threadId: thread.id });
34+
const detail = Option.getOrNull(state.data);
35+
const supported =
36+
useAtomValue(environmentServerConfigsAtom).get(environmentId)?.environment.capabilities
37+
.threadUsageLimitResume === true;
38+
const resume = useAtomCommand(threadEnvironment.resumeAfterUsageLimit, { reportFailure: false });
39+
const [busy, setBusy] = useState(false);
40+
const recovery = deriveUsageLimitRecovery({
41+
activities: detail?.activities ?? [],
42+
latestUserMessageAt: thread.latestUserMessageAt,
43+
sessionStatus: thread.session?.status ?? null,
44+
});
45+
if (!recovery || !supported) return null;
46+
47+
const run = async (action: "now" | "schedule" | "cancel", resumeAt?: string) => {
48+
setBusy(true);
49+
try {
50+
const result = await resume({
51+
environmentId,
52+
input: {
53+
threadId: thread.id,
54+
errorActivityId: recovery.errorActivityId,
55+
action,
56+
...(resumeAt ? { resumeAt } : {}),
57+
},
58+
});
59+
if (result._tag === "Failure") {
60+
const error = Cause.squash(result.cause);
61+
Alert.alert(
62+
action === "cancel" ? "Could not cancel the resume" : "Could not resume",
63+
error instanceof Error && error.message.trim().length > 0
64+
? error.message
65+
: "Try again in a moment.",
66+
);
67+
}
68+
} finally {
69+
setBusy(false);
70+
}
71+
};
72+
73+
const nowMs = Date.now();
74+
const resetInFuture = recovery.resetsAt !== null && Date.parse(recovery.resetsAt) > nowMs;
75+
const resumeAt = resetInFuture ? usageLimitResumeAt(recovery.resetsAt, nowMs) : null;
76+
const status =
77+
recovery.scheduledResumeAt !== null
78+
? `Resumes automatically: ${RESET_TIME_FORMATTER.format(Date.parse(recovery.scheduledResumeAt))}`
79+
: resetInFuture && recovery.resetsAt !== null
80+
? `Limit resets: ${RESET_TIME_FORMATTER.format(Date.parse(recovery.resetsAt))}`
81+
: null;
82+
83+
return (
84+
<View
85+
accessibilityRole="alert"
86+
className="mx-3 mb-2 gap-2 rounded-xl border border-border-subtle bg-composer-panel p-3"
87+
>
88+
<Text className="text-sm font-t3-medium text-foreground">{recovery.message}</Text>
89+
{status ? <Text className="text-xs text-foreground-muted">{status}</Text> : null}
90+
<View className="flex-row flex-wrap gap-2">
91+
{recovery.scheduledResumeAt !== null ? (
92+
<NoticeButton label="Cancel auto-resume" disabled={busy} onPress={() => run("cancel")} />
93+
) : (
94+
<>
95+
<NoticeButton label="Resume now" disabled={busy} onPress={() => run("now")} />
96+
{resumeAt !== null ? (
97+
<NoticeButton
98+
primary
99+
label="Resume when available"
100+
disabled={busy}
101+
onPress={() => run("schedule", resumeAt)}
102+
/>
103+
) : null}
104+
</>
105+
)}
106+
</View>
107+
</View>
108+
);
109+
}
110+
111+
function NoticeButton(props: {
112+
label: string;
113+
primary?: boolean;
114+
disabled: boolean;
115+
onPress: () => void;
116+
}) {
117+
return (
118+
<Pressable
119+
accessibilityRole="button"
120+
disabled={props.disabled}
121+
className={
122+
props.primary
123+
? "min-h-11 justify-center rounded-lg bg-primary px-3"
124+
: "min-h-11 justify-center rounded-lg border border-border-subtle px-3"
125+
}
126+
onPress={props.onPress}
127+
>
128+
<Text
129+
className={
130+
props.primary
131+
? "text-sm font-t3-medium text-primary-foreground"
132+
: "text-sm font-t3-medium text-foreground"
133+
}
134+
>
135+
{props.label}
136+
</Text>
137+
</Pressable>
138+
);
139+
}

‎apps/server/integration/OrchestrationEngineHarness.integration.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@ import * as ThreadSettlementReactor from "../src/orchestration/ThreadSettlementR
7070
import * as PullRequestSyncReactor from "../src/orchestration/PullRequestSyncReactor.ts";
7171
import * as ThreadPullRequestReactor from "../src/orchestration/ThreadPullRequestReactor.ts";
7272
import { ThreadTabSettlementReactor } from "../src/threadTabs/settlement.ts";
73+
import { UsageLimitResumeReactor } from "../src/orchestration/UsageLimitResumeReactor.ts";
7374
import { OrchestrationReactor } from "../src/orchestration/Services/OrchestrationReactor.ts";
7475
import { ProjectionSnapshotQuery } from "../src/orchestration/Services/ProjectionSnapshotQuery.ts";
7576
import {
@@ -417,6 +418,11 @@ export const makeOrchestrationIntegrationHarness = (
417418
drain: Effect.void,
418419
}),
419420
),
421+
Layer.provideMerge(
422+
Layer.succeed(UsageLimitResumeReactor, {
423+
start: () => Effect.void,
424+
}),
425+
),
420426
Layer.provideMerge(
421427
Layer.succeed(PullRequestSyncReactor.PullRequestSyncReactor, {
422428
start: () => Effect.void,

‎apps/server/src/environment/ServerEnvironment.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -231,6 +231,7 @@ export const make = Effect.gen(function* () {
231231
threadRestartContinuation: true,
232232
projectSettingsOverrides: true,
233233
threadSnooze: true,
234+
threadUsageLimitResume: true,
234235
environmentThemes: true,
235236
usageLimitSources: true,
236237
usagePriceOverrides: true,

‎apps/server/src/orchestration/Layers/OrchestrationReactor.test.ts‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import { makeOrchestrationReactor } from "./OrchestrationReactor.ts";
1717
import * as AgentAwarenessRelay from "../../relay/AgentAwarenessRelay.ts";
1818
import { StorageCleanup } from "../../storageCleanup.ts";
1919
import { ThreadTabSettlementReactor } from "../../threadTabs/settlement.ts";
20+
import { UsageLimitResumeReactor } from "../UsageLimitResumeReactor.ts";
2021

2122
describe("OrchestrationReactor", () => {
2223
let runtime: ManagedRuntime.ManagedRuntime<OrchestrationReactor, never> | null = null;
@@ -105,6 +106,14 @@ describe("OrchestrationReactor", () => {
105106
drain: Effect.void,
106107
}),
107108
),
109+
Layer.provideMerge(
110+
Layer.succeed(UsageLimitResumeReactor, {
111+
start: () => {
112+
started.push("usage-limit-resume-reactor");
113+
return Effect.void;
114+
},
115+
}),
116+
),
108117
Layer.provideMerge(
109118
Layer.succeed(PullRequestSyncReactor.PullRequestSyncReactor, {
110119
start: () => {
@@ -140,6 +149,7 @@ describe("OrchestrationReactor", () => {
140149
"thread-pull-request-reactor",
141150
"thread-settlement-reactor",
142151
"thread-tab-settlement-reactor",
152+
"usage-limit-resume-reactor",
143153
"pull-request-sync-reactor",
144154
"agent-awareness-relay",
145155
"storage-cleanup",

‎apps/server/src/orchestration/Layers/OrchestrationReactor.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import * as ThreadPullRequestReactor from "../ThreadPullRequestReactor.ts";
1515
import * as AgentAwarenessRelay from "../../relay/AgentAwarenessRelay.ts";
1616
import * as StorageCleanup from "../../storageCleanup.ts";
1717
import * as ThreadTabSettlementReactor from "../../threadTabs/settlement.ts";
18+
import * as UsageLimitResumeReactor from "../UsageLimitResumeReactor.ts";
1819

1920
export const makeOrchestrationReactor = Effect.gen(function* () {
2021
const providerRuntimeIngestion = yield* ProviderRuntimeIngestionService;
@@ -23,6 +24,7 @@ export const makeOrchestrationReactor = Effect.gen(function* () {
2324
const threadDeletionReactor = yield* ThreadDeletionReactor;
2425
const threadSettlementReactor = yield* ThreadSettlementReactor.ThreadSettlementReactor;
2526
const threadTabSettlementReactor = yield* ThreadTabSettlementReactor.ThreadTabSettlementReactor;
27+
const usageLimitResumeReactor = yield* UsageLimitResumeReactor.UsageLimitResumeReactor;
2628
const pullRequestSyncReactor = yield* PullRequestSyncReactor.PullRequestSyncReactor;
2729
const threadPullRequestReactor = yield* ThreadPullRequestReactor.ThreadPullRequestReactor;
2830
const agentAwarenessRelay = yield* AgentAwarenessRelay.AgentAwarenessRelay;
@@ -36,6 +38,7 @@ export const makeOrchestrationReactor = Effect.gen(function* () {
3638
yield* threadPullRequestReactor.start();
3739
yield* threadSettlementReactor.start();
3840
yield* threadTabSettlementReactor.start();
41+
yield* usageLimitResumeReactor.start();
3942
yield* pullRequestSyncReactor.start();
4043
yield* agentAwarenessRelay.start();
4144
yield* storageCleanup.start();

‎apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.test.ts‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4037,6 +4037,34 @@ describe("ProviderRuntimeIngestion", () => {
40374037
expect(activity?.kind).toBe("runtime.error");
40384038
expect(activityPayload?.message).toBe("runtime activity exploded");
40394039
expect(activityPayload?.code).toBe("subscription_sharing_usage_limit_exceeded");
4040+
expect(activityPayload).not.toHaveProperty("usageLimit");
4041+
});
4042+
4043+
it("keeps the usage-limit marker on the runtime.error activity", async () => {
4044+
const harness = await createHarness();
4045+
4046+
harness.emit({
4047+
type: "runtime.error",
4048+
eventId: asEventId("evt-usage-limit-error"),
4049+
provider: ProviderDriverKind.make("codex"),
4050+
createdAt: "2026-01-01T00:00:00.000Z",
4051+
threadId: asThreadId("thread-1"),
4052+
turnId: asTurnId("turn-usage-limit"),
4053+
payload: {
4054+
message: "Codex usage limit reached.",
4055+
usageLimit: { resetsAt: "2026-01-01T03:00:00.000Z" },
4056+
},
4057+
});
4058+
4059+
const thread = await waitForThread(harness.readModel, (entry) =>
4060+
entry.activities.some((activity) => activity.id === "evt-usage-limit-error"),
4061+
);
4062+
const activity = thread.activities.find(
4063+
(entry: ProviderRuntimeTestActivity) => entry.id === "evt-usage-limit-error",
4064+
);
4065+
expect(activity?.payload).toMatchObject({
4066+
usageLimit: { resetsAt: "2026-01-01T03:00:00.000Z" },
4067+
});
40404068
});
40414069

40424070
it("keeps the session running when a runtime.warning arrives during an active turn", async () => {

‎apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -573,6 +573,8 @@ export function runtimeEventToActivities(
573573
payload: {
574574
message: truncateDetail(event.payload.message),
575575
...(event.payload.code ? { code: event.payload.code } : {}),
576+
// Clients offer usage-limit recovery from this; see usageLimitRecovery.
577+
...(event.payload.usageLimit ? { usageLimit: event.payload.usageLimit } : {}),
576578
},
577579
turnId: toTurnId(event.turnId) ?? null,
578580
...maybeSequence,

0 commit comments

Comments
 (0)