Skip to content

Commit e58a4bc

Browse files
fix(server): limited turns through CLIProxyAPI resume when the hub refills
CLIProxyAPI reports a spent pool as a 429 with a reset time, but Claude Code and Codex drop that time, so T3 marked the thread Limited with no reset and Resume at reset, auto-resume, and snooze could not start. When a usage-limit terminal has no reset time, the ingestor now asks the connected CLIProxyAPI hubs. If every account a hub pools for that provider is spent, the reset is when the first one refills. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent dac8b11 commit e58a4bc

6 files changed

Lines changed: 164 additions & 7 deletions

File tree

‎apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts‎

Lines changed: 28 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ import * as ProjectionStore from "./ProjectionStore.ts";
4747
import * as ProviderEventIngestor from "./ProviderEventIngestor.ts";
4848
import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts";
4949
import * as ProviderInstanceRegistry from "../provider/ProviderInstanceRegistry.ts";
50+
import * as UsageLimitSources from "../usage/UsageLimitSources.ts";
5051
import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
5152
import type { ProviderInstance } from "@t3tools/provider-core/server/driver";
5253
import { makeProviderFailure } from "@t3tools/provider-core/server/failure";
@@ -233,6 +234,16 @@ it.effect("refreshes the failing instance snapshot after a committed usage_limit
233234
Effect.scoped(
234235
Effect.gen(function* () {
235236
const refreshed = yield* Deferred.make<void>();
237+
const hubResetAt = "2026-10-30T05:00:00.000Z";
238+
const nativeResetAt = "2026-10-30T03:00:00.000Z";
239+
const fallbackDrivers: ProviderDriverKind[] = [];
240+
const layerResetFallback = Layer.succeed(UsageLimitSources.UsageLimitResetFallback, {
241+
resetAt: (driver) =>
242+
Effect.sync(() => {
243+
fallbackDrivers.push(driver);
244+
return hubResetAt;
245+
}),
246+
});
236247
const adapter = yield* ProviderAdapter.ProviderAdapterV2.pipe(
237248
Effect.provide(
238249
Layer.mock(ProviderAdapter.ProviderAdapterV2)({
@@ -290,7 +301,11 @@ it.effect("refreshes the failing instance snapshot after a committed usage_limit
290301
providerInstanceId: instance.instanceId,
291302
threadId: threadEvent.threadId,
292303
};
293-
for (const failureClass of ["validation_error", "usage_limit"] as const) {
304+
for (const [failureClass, resetAt, expectedResetAt] of [
305+
["validation_error", undefined, undefined],
306+
["usage_limit", undefined, hubResetAt],
307+
["usage_limit", nativeResetAt, nativeResetAt],
308+
] as const) {
294309
const stored = yield* ingestor.ingestNormalized({
295310
...input,
296311
event: {
@@ -302,24 +317,33 @@ it.effect("refreshes the failing instance snapshot after a committed usage_limit
302317
}),
303318
providerTurnId: idAllocator.derive.providerTurn({
304319
driver: instance.driverKind,
305-
nativeTurnId: failureClass,
320+
nativeTurnId: `${failureClass}:${resetAt ?? "unknown"}`,
306321
}),
307322
runOrdinal: 1,
308323
failureItemOrdinal: 1,
309324
status: "failed",
310325
failure: makeProviderFailure({
311326
class: failureClass,
312327
message: "Provider rejected turn.",
328+
...(resetAt === undefined ? {} : { resetAt }),
313329
}),
314330
threadDisposition: "reusable",
315331
},
316332
});
317333
assert.equal(stored.length, 1);
334+
const event = stored[0]?.event;
335+
const item = event?.type === "turn-item.updated" ? event.payload : undefined;
336+
assert.equal(item?.type === "error" ? item.failure.resetAt : null, expectedResetAt);
318337
if (failureClass === "validation_error") assert.deepEqual(lookups, []);
319338
}
320-
assert.deepEqual(lookups, [instance.instanceId]);
339+
assert.deepEqual(lookups, [instance.instanceId, instance.instanceId]);
340+
assert.deepEqual(fallbackDrivers, [instance.driverKind]);
321341
yield* Deferred.await(refreshed);
322-
}).pipe(Effect.provide(makeLayerTest(layerTestEventSink, instances)));
342+
}).pipe(
343+
Effect.provide(
344+
makeLayerTest(layerTestEventSink, instances).pipe(Layer.provide(layerResetFallback)),
345+
),
346+
);
323347
}),
324348
),
325349
);

‎apps/server/src/orchestration-v2/ProviderEventIngestor.ts‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ import { getModelSelectionStringOptionValue, modelSelectionsEqual } from "@t3too
3535
import * as AnalyticsService from "../telemetry/AnalyticsService.ts";
3636
import * as GitVcsDriver from "../vcs/GitVcsDriver.ts";
3737
import * as ProviderInstanceRegistry from "../provider/ProviderInstanceRegistry.ts";
38+
import { UsageLimitResetFallback } from "../usage/UsageLimitSources.ts";
3839
import * as EventSink from "./EventSink.ts";
3940
import * as ProjectionStore from "./ProjectionStore.ts";
4041
import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
@@ -339,6 +340,7 @@ export const layer: Layer.Layer<
339340
const analytics = yield* ProviderTurnAnalytics;
340341
const agentScope = yield* AgentScope;
341342
const instances = yield* ProviderInstanceRegistry.ProviderInstanceRegistry;
343+
const usageLimitResets = yield* UsageLimitResetFallback;
342344
const scope = yield* Effect.scope;
343345
const completedTurnAnalytics = new Set<string>();
344346

@@ -709,9 +711,15 @@ export const layer: Layer.Layer<
709711
return dismissed;
710712
}
711713
const occurredAt = yield* DateTime.now;
712-
const failure = (yield* agentScope.oomKilled(input.threadId))
714+
const reported = (yield* agentScope.oomKilled(input.threadId))
713715
? { ...input.event.failure, message: "Killed: out of memory.", code: "oom_kill" }
714716
: input.event.failure;
717+
const fallbackResetAt =
718+
reported.class === "usage_limit" && reported.resetAt == null
719+
? yield* usageLimitResets.resetAt(input.event.driver)
720+
: null;
721+
const failure =
722+
fallbackResetAt === null ? reported : { ...reported, resetAt: fallbackResetAt };
715723
return [
716724
...dismissed,
717725
yield* makeDomainEvent(input, {

‎apps/server/src/server.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -520,6 +520,7 @@ const layerScheduledTaskWebhookOrigin = Layer.effect(
520520
const layerOrchestrationV2Runtime = RuntimeLayer.layerProduction.pipe(
521521
Layer.provide(layerScheduledTaskWebhookOrigin),
522522
Layer.provide(ProviderEventIngestor.layerAnalytics),
523+
Layer.provide(UsageLimitSources.layerResetFallback),
523524
Layer.provide(layerCheckpointStore),
524525
Layer.provide(layerGitWorkflow),
525526
Layer.provide(

‎apps/server/src/usage/UsageLimitSources.test.ts‎

Lines changed: 71 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,13 @@
11
import * as NodeCrypto from "@effect/platform-node/NodeCrypto";
2-
import { expect, it } from "@effect/vitest";
3-
import { ServerConfigStreamEvent, UsageLimitSourceId } from "@t3tools/contracts";
2+
import { describe, expect, it } from "@effect/vitest";
3+
import {
4+
ProviderDriverKind,
5+
ServerConfigStreamEvent,
6+
UsageLimitSourceId,
7+
type ServerProviderUsageWindow,
8+
type UsageLimitSourceSnapshot,
9+
} from "@t3tools/contracts";
10+
import * as DateTime from "effect/DateTime";
411
import * as Effect from "effect/Effect";
512
import * as Layer from "effect/Layer";
613
import * as Schema from "effect/Schema";
@@ -83,3 +90,65 @@ it.effect("keeps malformed source URLs encodable for config subscribers", () =>
8390
Effect.scoped,
8491
),
8592
);
93+
94+
describe("hubPoolResetAt", () => {
95+
const now = Date.parse("2026-10-30T00:00:00.000Z");
96+
const hours = (count: number) => DateTime.formatIso(DateTime.makeUnsafe(now + count * 3_600_000));
97+
const window = (usedPercent: number, resetsAt?: string): ServerProviderUsageWindow => ({
98+
id: resetsAt ?? `window-${usedPercent}`,
99+
kind: "session",
100+
label: "5h",
101+
usedPercent,
102+
...(resetsAt === undefined ? {} : { resetsAt }),
103+
});
104+
const hub = (
105+
...accounts: ReadonlyArray<readonly [string, ReadonlyArray<ServerProviderUsageWindow>]>
106+
): UsageLimitSourceSnapshot => ({
107+
id: UsageLimitSourceId.make("hub"),
108+
kind: "cliproxy",
109+
label: "hub",
110+
checkedAt: hours(0),
111+
accounts: accounts.map(([driver, windows], index) => ({
112+
id: `account-${index}`,
113+
driver: ProviderDriverKind.make(driver),
114+
usageLimits: { checkedAt: hours(0), windows },
115+
})),
116+
});
117+
const codex = ProviderDriverKind.make("codex");
118+
119+
it.each([
120+
{
121+
name: "the first spent account to recover, after all its spent windows reset",
122+
sources: [
123+
hub(
124+
["codex", [window(100, hours(2))]],
125+
["codex", [window(100, hours(1)), window(100, hours(3))]],
126+
),
127+
hub(["claudeAgent", [window(100, hours(0.5))]]),
128+
],
129+
expected: hours(2),
130+
},
131+
{
132+
name: "nothing while an account still has quota",
133+
sources: [hub(["codex", [window(100, hours(2))]], ["codex", [window(40, hours(1))]])],
134+
expected: null,
135+
},
136+
{
137+
name: "nothing once a spent window's reset has passed",
138+
sources: [hub(["codex", [window(100, hours(2))]], ["codex", [window(100, hours(-1))]])],
139+
expected: null,
140+
},
141+
{
142+
name: "a recovery time while skipping accounts without readings or reset times",
143+
sources: [hub(["codex", []], ["codex", [window(100)]], ["codex", [window(100, hours(4))]])],
144+
expected: hours(4),
145+
},
146+
{
147+
name: "nothing when the hub pools none of the driver's accounts",
148+
sources: [hub(["claudeAgent", [window(100, hours(1))]])],
149+
expected: null,
150+
},
151+
])("returns $name", ({ sources, expected }) => {
152+
expect(UsageLimitSources.hubPoolResetAt(sources, codex, now)).toBe(expected);
153+
});
154+
});

‎apps/server/src/usage/UsageLimitSources.ts‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import {
33
UsageLimitSourceError,
44
type UsageLimitSourceConsumeResetCreditInput,
55
type ProviderConsumeResetCreditResult,
6+
type ProviderDriverKind,
67
type ServerSettings,
78
type UsageLimitSourceConfig,
89
type UsageLimitSourceId,
@@ -168,3 +169,52 @@ export const make = Effect.gen(function* () {
168169
});
169170

170171
export const layer = Layer.effect(UsageLimitSources, make);
172+
173+
/**
174+
* A hub answers for a 429 only when every account it pools for the driver is spent;
175+
* the pool recovers when its first account does.
176+
*/
177+
export function hubPoolResetAt(
178+
sources: ReadonlyArray<UsageLimitSourceSnapshot>,
179+
driver: ProviderDriverKind,
180+
nowMs: number,
181+
): string | null {
182+
let earliest: number | null = null;
183+
for (const account of sources.flatMap((source) => source.accounts)) {
184+
if (account.driver !== driver) continue;
185+
const { windows } = account.usageLimits;
186+
const exhausted = windows.filter((window) => window.usedPercent >= 100);
187+
if (exhausted.length === 0) {
188+
if (windows.length > 0) return null;
189+
continue;
190+
}
191+
const resets = exhausted.map((window) => Date.parse(window.resetsAt ?? ""));
192+
if (resets.some((reset) => !Number.isFinite(reset))) continue;
193+
const recoversAt = Math.max(...resets);
194+
if (recoversAt <= nowMs) return null;
195+
earliest = earliest === null ? recoversAt : Math.min(earliest, recoversAt);
196+
}
197+
return earliest === null ? null : DateTime.formatIso(DateTime.makeUnsafe(earliest));
198+
}
199+
200+
/** Supplies a reset time for usage limits a provider reports without one. */
201+
export class UsageLimitResetFallback extends Context.Reference<{
202+
readonly resetAt: (driver: ProviderDriverKind) => Effect.Effect<string | null>;
203+
}>("t3/usage/UsageLimitResetFallback", {
204+
defaultValue: () => ({ resetAt: () => Effect.succeed(null) }),
205+
}) {}
206+
207+
export const layerResetFallback = Layer.effect(
208+
UsageLimitResetFallback,
209+
Effect.gen(function* () {
210+
const sources = yield* UsageLimitSources;
211+
return {
212+
resetAt: Effect.fn("UsageLimitResetFallback.resetAt")(function* (driver: ProviderDriverKind) {
213+
// The terminal waits on this read, so a slow hub falls back to its last snapshot.
214+
yield* sources.refresh.pipe(Effect.timeoutOption("5 seconds"));
215+
const now = yield* DateTime.now;
216+
return hubPoolResetAt(yield* sources.current, driver, DateTime.toEpochMillis(now));
217+
}),
218+
};
219+
}),
220+
);

‎docs/user/usage.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,11 @@ This connection supplies usage information; configure
146146
the provider separately to send agent requests through the hub. Remove the hub from the same
147147
settings section when you no longer need it.
148148

149+
CLIProxyAPI does not pass its reset time through Claude Code or Codex. When a Claude or Codex
150+
turn stops on a usage limit without one, and every account the hub pools for that provider is
151+
spent, the thread resets when the first of those accounts refills. **Resume at reset**, automatic
152+
resume, and snooze then work as they do for direct subscriptions.
153+
149154
## Subscription usage widget
150155

151156
Add **Subscription usage** from your iOS or Android widget gallery to see remaining Codex and

0 commit comments

Comments
 (0)