Skip to content

Commit d64335b

Browse files
fix(codex): name the usage limit and its reset instead of relaying "out of credits" (#10473)
Co-authored-by: Claude Fable 5.1 <[email protected]>
1 parent 2c8e95a commit d64335b

5 files changed

Lines changed: 537 additions & 2 deletions

File tree

‎apps/server/src/provider/Layers/CodexAdapter.test.ts‎

Lines changed: 299 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2694,3 +2694,302 @@ it.effect("flushes managed native logs when the adapter layer shuts down", () =>
26942694
}
26952695
}),
26962696
);
2697+
2698+
const usageLimitRuntimeFactory = makeRuntimeFactory();
2699+
const usageLimitLayer = it.layer(
2700+
Layer.effect(
2701+
CodexAdapter,
2702+
Effect.gen(function* () {
2703+
const codexConfig = decodeCodexSettings({});
2704+
return yield* makeCodexAdapter(codexConfig, {
2705+
makeRuntime: usageLimitRuntimeFactory.factory,
2706+
});
2707+
}),
2708+
).pipe(
2709+
Layer.provideMerge(ServerConfig.layerTest(process.cwd(), process.cwd())),
2710+
Layer.provideMerge(ServerSettingsService.layerTest()),
2711+
Layer.provideMerge(providerSessionDirectoryTestLayer),
2712+
Layer.provideMerge(NodeServices.layer),
2713+
),
2714+
);
2715+
2716+
const USAGE_LIMIT_NOW = "2026-01-01T00:00:00.000Z";
2717+
const USAGE_LIMIT_NOW_SECONDS = Date.parse(USAGE_LIMIT_NOW) / 1000;
2718+
const CODEX_OUT_OF_CREDITS =
2719+
"Your workspace is out of credits. Ask your workspace owner to refill in order to continue.";
2720+
2721+
function startUsageLimitRuntime() {
2722+
return Effect.gen(function* () {
2723+
const adapter = yield* CodexAdapter;
2724+
yield* adapter.startSession({
2725+
provider: ProviderDriverKind.make("codex"),
2726+
threadId: asThreadId("thread-1"),
2727+
runtimeMode: "full-access",
2728+
});
2729+
const runtime = usageLimitRuntimeFactory.lastRuntime;
2730+
NodeAssert.ok(runtime);
2731+
return { adapter, runtime };
2732+
});
2733+
}
2734+
2735+
function codexErrorNotification(input: {
2736+
readonly id: string;
2737+
readonly message: string;
2738+
readonly codexErrorInfo?: string;
2739+
}): ProviderEvent {
2740+
return {
2741+
id: asEventId(input.id),
2742+
kind: "notification",
2743+
provider: ProviderDriverKind.make("codex"),
2744+
threadId: asThreadId("thread-1"),
2745+
turnId: asTurnId("turn-limit"),
2746+
createdAt: USAGE_LIMIT_NOW,
2747+
method: "error",
2748+
payload: {
2749+
threadId: "thread-1",
2750+
turnId: "turn-limit",
2751+
willRetry: false,
2752+
error: {
2753+
message: input.message,
2754+
...(input.codexErrorInfo ? { codexErrorInfo: input.codexErrorInfo } : {}),
2755+
},
2756+
},
2757+
};
2758+
}
2759+
2760+
function codexRateLimitsNotification(input: {
2761+
readonly id: string;
2762+
readonly rateLimitReachedType?: string;
2763+
readonly primary?: { readonly usedPercent: number; readonly resetsInSeconds: number };
2764+
readonly secondary?: { readonly usedPercent: number; readonly resetsInSeconds: number };
2765+
}): ProviderEvent {
2766+
return {
2767+
id: asEventId(input.id),
2768+
kind: "notification",
2769+
provider: ProviderDriverKind.make("codex"),
2770+
threadId: asThreadId("thread-1"),
2771+
turnId: asTurnId("turn-limit"),
2772+
createdAt: USAGE_LIMIT_NOW,
2773+
method: "account/rateLimits/updated",
2774+
payload: {
2775+
rateLimits: {
2776+
limitId: "codex",
2777+
...(input.rateLimitReachedType ? { rateLimitReachedType: input.rateLimitReachedType } : {}),
2778+
...(input.primary
2779+
? {
2780+
primary: {
2781+
usedPercent: input.primary.usedPercent,
2782+
resetsAt: USAGE_LIMIT_NOW_SECONDS + input.primary.resetsInSeconds,
2783+
windowDurationMins: 300,
2784+
},
2785+
}
2786+
: {}),
2787+
...(input.secondary
2788+
? {
2789+
secondary: {
2790+
usedPercent: input.secondary.usedPercent,
2791+
resetsAt: USAGE_LIMIT_NOW_SECONDS + input.secondary.resetsInSeconds,
2792+
windowDurationMins: 10_080,
2793+
},
2794+
}
2795+
: {}),
2796+
},
2797+
},
2798+
};
2799+
}
2800+
2801+
function codexUsageLimitTurnFailed(id: string, turnId = "turn-limit"): ProviderEvent {
2802+
return {
2803+
id: asEventId(id),
2804+
kind: "notification",
2805+
provider: ProviderDriverKind.make("codex"),
2806+
threadId: asThreadId("thread-1"),
2807+
turnId: asTurnId(turnId),
2808+
createdAt: USAGE_LIMIT_NOW,
2809+
method: "turn/completed",
2810+
payload: {
2811+
threadId: "thread-1",
2812+
turn: {
2813+
id: turnId,
2814+
items: [],
2815+
status: "failed",
2816+
error: { message: CODEX_OUT_OF_CREDITS, codexErrorInfo: "usageLimitExceeded" },
2817+
},
2818+
},
2819+
};
2820+
}
2821+
2822+
usageLimitLayer("CodexAdapterLive usage limits", (it) => {
2823+
it.effect("names the exhausted window and the workspace's missing credits", () =>
2824+
Effect.gen(function* () {
2825+
const { adapter, runtime } = yield* startUsageLimitRuntime();
2826+
const eventsFiber = yield* adapter.streamEvents.pipe(
2827+
Stream.take(5),
2828+
Stream.runCollect,
2829+
Effect.forkChild,
2830+
);
2831+
2832+
yield* runtime.emit(
2833+
codexErrorNotification({
2834+
id: "evt-limit-error",
2835+
message: CODEX_OUT_OF_CREDITS,
2836+
codexErrorInfo: "usageLimitExceeded",
2837+
}),
2838+
);
2839+
yield* runtime.emit(
2840+
codexRateLimitsNotification({
2841+
id: "evt-limit-rate-limits",
2842+
rateLimitReachedType: "workspace_owner_credits_depleted",
2843+
primary: { usedPercent: 40, resetsInSeconds: 3_600 },
2844+
secondary: { usedPercent: 100, resetsInSeconds: 5 * 86_400 + 5 * 3_600 },
2845+
}),
2846+
);
2847+
yield* runtime.emit(codexUsageLimitTurnFailed("evt-limit-turn"));
2848+
// A second turn stopping on the same limit says as much as the first.
2849+
yield* runtime.emit(codexUsageLimitTurnFailed("evt-limit-turn-2", "turn-limit-2"));
2850+
2851+
const events = Array.from(yield* Fiber.join(eventsFiber));
2852+
const expected =
2853+
"Codex usage limit reached. The weekly limit resets in 5d 5h. The workspace has no credits to continue sooner: ask your workspace owner to add credits, or send the message again once the limit resets.";
2854+
NodeAssert.deepStrictEqual(
2855+
events.map((event) => event.type),
2856+
[
2857+
"account.rate-limits.updated",
2858+
"runtime.error",
2859+
"turn.completed",
2860+
"runtime.error",
2861+
"turn.completed",
2862+
],
2863+
);
2864+
for (const event of events) {
2865+
if (event.type === "runtime.error") {
2866+
NodeAssert.equal(event.payload.message, expected);
2867+
NodeAssert.equal(event.payload.detail, CODEX_OUT_OF_CREDITS);
2868+
}
2869+
if (event.type === "turn.completed") {
2870+
NodeAssert.equal(event.payload.errorMessage, expected);
2871+
}
2872+
}
2873+
}),
2874+
);
2875+
2876+
it.effect("names the session window for a plan limit", () =>
2877+
Effect.gen(function* () {
2878+
const { adapter, runtime } = yield* startUsageLimitRuntime();
2879+
const eventsFiber = yield* adapter.streamEvents.pipe(
2880+
Stream.take(3),
2881+
Stream.runCollect,
2882+
Effect.forkChild,
2883+
);
2884+
2885+
yield* runtime.emit(
2886+
codexErrorNotification({
2887+
id: "evt-plan-error",
2888+
message: "You've hit your usage limit.",
2889+
codexErrorInfo: "usageLimitExceeded",
2890+
}),
2891+
);
2892+
yield* runtime.emit(
2893+
codexRateLimitsNotification({
2894+
id: "evt-plan-rate-limits",
2895+
rateLimitReachedType: "rate_limit_reached",
2896+
primary: { usedPercent: 100, resetsInSeconds: 3 * 3_600 + 20 * 60 },
2897+
}),
2898+
);
2899+
yield* runtime.emit(codexUsageLimitTurnFailed("evt-plan-turn"));
2900+
2901+
const events = Array.from(yield* Fiber.join(eventsFiber));
2902+
const completed = events.find((event) => event.type === "turn.completed");
2903+
NodeAssert.equal(
2904+
completed?.payload.errorMessage,
2905+
"Codex usage limit reached. The session limit resets in 3h 20m. Send the message again once the limit resets.",
2906+
);
2907+
}),
2908+
);
2909+
2910+
it.effect("reads a rate-limit snapshot seen earlier in the session", () =>
2911+
Effect.gen(function* () {
2912+
const { adapter, runtime } = yield* startUsageLimitRuntime();
2913+
const eventsFiber = yield* adapter.streamEvents.pipe(
2914+
Stream.take(3),
2915+
Stream.runCollect,
2916+
Effect.forkChild,
2917+
);
2918+
2919+
// The window arrives long before the stop, and the update that reports the
2920+
// limit as reached carries no windows of its own.
2921+
yield* runtime.emit(
2922+
codexRateLimitsNotification({
2923+
id: "evt-early-rate-limits",
2924+
primary: { usedPercent: 100, resetsInSeconds: 3 * 3_600 + 20 * 60 },
2925+
}),
2926+
);
2927+
yield* runtime.emit(
2928+
codexRateLimitsNotification({
2929+
id: "evt-sparse-rate-limits",
2930+
rateLimitReachedType: "rate_limit_reached",
2931+
}),
2932+
);
2933+
yield* runtime.emit(codexUsageLimitTurnFailed("evt-early-turn"));
2934+
2935+
const events = Array.from(yield* Fiber.join(eventsFiber));
2936+
const completed = events.find((event) => event.type === "turn.completed");
2937+
NodeAssert.equal(
2938+
completed?.payload.errorMessage,
2939+
"Codex usage limit reached. The session limit resets in 3h 20m. Send the message again once the limit resets.",
2940+
);
2941+
}),
2942+
);
2943+
2944+
it.effect("falls back to the short message without a rate-limit snapshot", () =>
2945+
Effect.gen(function* () {
2946+
const { adapter, runtime } = yield* startUsageLimitRuntime();
2947+
const eventsFiber = yield* adapter.streamEvents.pipe(
2948+
Stream.take(2),
2949+
Stream.runCollect,
2950+
Effect.forkChild,
2951+
);
2952+
2953+
yield* runtime.emit(
2954+
codexErrorNotification({
2955+
id: "evt-bare-error",
2956+
message: CODEX_OUT_OF_CREDITS,
2957+
codexErrorInfo: "usageLimitExceeded",
2958+
}),
2959+
);
2960+
yield* runtime.emit(codexUsageLimitTurnFailed("evt-bare-turn"));
2961+
2962+
const events = Array.from(yield* Fiber.join(eventsFiber));
2963+
const expected = "Codex usage limit reached. Send the message again once the limit resets.";
2964+
NodeAssert.deepStrictEqual(
2965+
events.map((event) => event.type),
2966+
["runtime.error", "turn.completed"],
2967+
);
2968+
const runtimeError = events.find((event) => event.type === "runtime.error");
2969+
NodeAssert.equal(runtimeError?.payload.message, expected);
2970+
const completed = events.find((event) => event.type === "turn.completed");
2971+
NodeAssert.equal(completed?.payload.errorMessage, expected);
2972+
}),
2973+
);
2974+
2975+
it.effect("still relays other provider errors as they arrive", () =>
2976+
Effect.gen(function* () {
2977+
const { adapter, runtime } = yield* startUsageLimitRuntime();
2978+
const firstEventFiber = yield* Stream.runHead(adapter.streamEvents).pipe(Effect.forkChild);
2979+
2980+
yield* runtime.emit(
2981+
codexErrorNotification({
2982+
id: "evt-other-error",
2983+
message: "Codex is temporarily unavailable.",
2984+
codexErrorInfo: "internalServerError",
2985+
}),
2986+
);
2987+
2988+
const first = yield* Fiber.join(firstEventFiber);
2989+
NodeAssert.equal(first._tag, "Some");
2990+
if (first._tag !== "Some" || first.value.type !== "runtime.error") return;
2991+
NodeAssert.equal(first.value.payload.message, "Codex is temporarily unavailable.");
2992+
NodeAssert.equal(first.value.payload.class, "provider_error");
2993+
}),
2994+
);
2995+
});

‎apps/server/src/provider/Layers/CodexAdapter.ts‎

Lines changed: 60 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,12 @@ import {
7171
} from "./CodexSessionRuntime.ts";
7272
import { type EventNdjsonLogger, makeEventNdjsonLogger } from "./EventNdjsonLogger.ts";
7373
import { resolveCodexLaunchArgs } from "./codexLaunchArgs.ts";
74-
import { codexRateLimitsToUpdate } from "./codexUsageLimits.ts";
74+
import {
75+
type CodexRateLimitSnapshot,
76+
codexRateLimitsToUpdate,
77+
codexUsageLimitMessage,
78+
mergeCodexRateLimits,
79+
} from "./codexUsageLimits.ts";
7580
const isCodexAppServerProcessExitedError = Schema.is(CodexErrors.CodexAppServerProcessExitedError);
7681
const isCodexAppServerTransportError = Schema.is(CodexErrors.CodexAppServerTransportError);
7782
const isCodexSessionRuntimeThreadIdMissingError = Schema.is(
@@ -2282,6 +2287,12 @@ export const makeCodexAdapter = Effect.fn("makeCodexAdapter")(function* (
22822287
: {}),
22832288
};
22842289
const turnTokenUsage = makeCodexTurnTokenUsageState();
2290+
// Codex reports a usage-limit stop as OpenAI's own sentence, which on a
2291+
// Business workspace blames credits for a window that ran out. The
2292+
// snapshot naming that window arrives in its own notification, before or
2293+
// after the stop and often sparse, so keep the session's merged view of
2294+
// it and read it when a turn fails on the limit.
2295+
let rateLimits: CodexRateLimitSnapshot | undefined;
22852296
const sessionScope = yield* Scope.make("sequential");
22862297
let sessionScopeTransferred = false;
22872298
yield* Effect.addFinalizer(() =>
@@ -2339,12 +2350,56 @@ export const makeCodexAdapter = Effect.fn("makeCodexAdapter")(function* (
23392350
}
23402351
}
23412352

2342-
const runtimeEvents = mapToRuntimeEvents(event, event.threadId).map((runtimeEvent) => {
2353+
if (event.method === "account/rateLimits/updated") {
2354+
const limitsPayload = readPayload(
2355+
EffectCodexSchema.V2AccountRateLimitsUpdatedNotification,
2356+
event.payload,
2357+
);
2358+
if (limitsPayload) {
2359+
rateLimits = mergeCodexRateLimits(rateLimits, limitsPayload.rateLimits);
2360+
}
2361+
} else if (event.method === "error") {
2362+
const errorPayload = readPayload(
2363+
EffectCodexSchema.V2ErrorNotification,
2364+
event.payload,
2365+
);
2366+
// The failed `turn/completed` repeats this sentence and is answered
2367+
// below; relaying both would show the limit twice.
2368+
if (errorPayload?.error.codexErrorInfo === "usageLimitExceeded") return;
2369+
}
2370+
2371+
let usageLimitError: ProviderRuntimeEvent | undefined;
2372+
let usageLimitMessage: string | undefined;
2373+
if (event.method === "turn/completed") {
2374+
const completedPayload = readPayload(
2375+
EffectCodexSchema.V2TurnCompletedNotification,
2376+
event.payload,
2377+
);
2378+
const turnError =
2379+
completedPayload?.turn.status === "failed"
2380+
? completedPayload.turn.error
2381+
: undefined;
2382+
if (turnError?.codexErrorInfo === "usageLimitExceeded") {
2383+
usageLimitMessage = codexUsageLimitMessage(rateLimits, event.createdAt);
2384+
usageLimitError = {
2385+
...runtimeEventBase(event, event.threadId),
2386+
type: "runtime.error",
2387+
payload: {
2388+
message: usageLimitMessage,
2389+
class: "provider_error",
2390+
...(turnError.message ? { detail: turnError.message } : {}),
2391+
},
2392+
};
2393+
}
2394+
}
2395+
2396+
const mappedEvents = mapToRuntimeEvents(event, event.threadId).map((runtimeEvent) => {
23432397
if (runtimeEvent.type === "turn.completed" && runtimeEvent.turnId) {
23442398
return {
23452399
...runtimeEvent,
23462400
payload: {
23472401
...runtimeEvent.payload,
2402+
...(usageLimitMessage ? { errorMessage: usageLimitMessage } : {}),
23482403
tokenUsage: completeCodexTurnTokenUsage(
23492404
turnTokenUsage,
23502405
String(runtimeEvent.turnId),
@@ -2368,6 +2423,9 @@ export const makeCodexAdapter = Effect.fn("makeCodexAdapter")(function* (
23682423
}
23692424
return runtimeEvent;
23702425
});
2426+
const runtimeEvents = usageLimitError
2427+
? [usageLimitError, ...mappedEvents]
2428+
: mappedEvents;
23712429
if (runtimeEvents.length === 0) {
23722430
yield* Effect.logDebug("ignoring unhandled Codex provider event", {
23732431
method: event.method,

0 commit comments

Comments
 (0)