Skip to content

Commit d5519db

Browse files
committed
fix(codex): reject ambiguous quota payload bytes
1 parent d4dcff5 commit d5519db

3 files changed

Lines changed: 142 additions & 6 deletions

File tree

‎src/codex/quota-rejection.ts‎

Lines changed: 93 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,93 @@ function hasOwnField(container: Record<string, unknown>, field: string): boolean
4949
return Object.prototype.hasOwnProperty.call(container, field);
5050
}
5151

52+
type JsonScanResult = {
53+
next: number;
54+
duplicate: boolean;
55+
};
56+
57+
function skipJsonWhitespace(text: string, index: number): number {
58+
while (index < text.length && /[\t\n\r ]/.test(text[index] ?? "")) index += 1;
59+
return index;
60+
}
61+
62+
function scanJsonStringEnd(text: string, index: number): number {
63+
if (text[index] !== '"') throw new SyntaxError("expected JSON string");
64+
for (let cursor = index + 1; cursor < text.length; cursor += 1) {
65+
const char = text[cursor];
66+
if (char === '"') return cursor + 1;
67+
if (char === "\\") cursor += 1;
68+
}
69+
throw new SyntaxError("unterminated JSON string");
70+
}
71+
72+
function scanJsonValue(text: string, index: number): JsonScanResult {
73+
const start = skipJsonWhitespace(text, index);
74+
if (text[start] === "{") return scanJsonObject(text, start);
75+
if (text[start] === "[") return scanJsonArray(text, start);
76+
if (text[start] === '"') return { next: scanJsonStringEnd(text, start), duplicate: false };
77+
78+
for (const literal of ["true", "false", "null"]) {
79+
if (text.startsWith(literal, start)) {
80+
return { next: start + literal.length, duplicate: false };
81+
}
82+
}
83+
const number = text.slice(start).match(/^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?/);
84+
if (!number) throw new SyntaxError("expected JSON value");
85+
return { next: start + number[0].length, duplicate: false };
86+
}
87+
88+
function scanJsonObject(text: string, index: number): JsonScanResult {
89+
const keys = new Set<string>();
90+
let duplicate = false;
91+
let cursor = skipJsonWhitespace(text, index + 1);
92+
if (text[cursor] === "}") return { next: cursor + 1, duplicate: false };
93+
94+
while (cursor < text.length) {
95+
const keyEnd = scanJsonStringEnd(text, cursor);
96+
const key = JSON.parse(text.slice(cursor, keyEnd)) as unknown;
97+
if (typeof key !== "string") throw new SyntaxError("invalid JSON object key");
98+
if (keys.has(key)) duplicate = true;
99+
keys.add(key);
100+
101+
cursor = skipJsonWhitespace(text, keyEnd);
102+
if (text[cursor] !== ":") throw new SyntaxError("expected JSON object colon");
103+
const value = scanJsonValue(text, cursor + 1);
104+
duplicate ||= value.duplicate;
105+
cursor = skipJsonWhitespace(text, value.next);
106+
if (text[cursor] === "}") return { next: cursor + 1, duplicate };
107+
if (text[cursor] !== ",") throw new SyntaxError("expected JSON object separator");
108+
cursor = skipJsonWhitespace(text, cursor + 1);
109+
}
110+
throw new SyntaxError("unterminated JSON object");
111+
}
112+
113+
function scanJsonArray(text: string, index: number): JsonScanResult {
114+
let duplicate = false;
115+
let cursor = skipJsonWhitespace(text, index + 1);
116+
if (text[cursor] === "]") return { next: cursor + 1, duplicate: false };
117+
118+
while (cursor < text.length) {
119+
const value = scanJsonValue(text, cursor);
120+
duplicate ||= value.duplicate;
121+
cursor = skipJsonWhitespace(text, value.next);
122+
if (text[cursor] === "]") return { next: cursor + 1, duplicate };
123+
if (text[cursor] !== ",") throw new SyntaxError("expected JSON array separator");
124+
cursor = skipJsonWhitespace(text, cursor + 1);
125+
}
126+
throw new SyntaxError("unterminated JSON array");
127+
}
128+
129+
function hasDuplicateJsonObjectKeys(text: string): boolean {
130+
try {
131+
const result = scanJsonValue(text, 0);
132+
return result.duplicate || skipJsonWhitespace(text, result.next) !== text.length;
133+
} catch {
134+
// Scanner disagreement is untrusted input, just like JSON.parse failure.
135+
return true;
136+
}
137+
}
138+
52139
function exactResetEligibleCode(
53140
container: Record<string, unknown>,
54141
): CodexResetEligibleExhaustionCode | undefined {
@@ -88,9 +175,13 @@ async function resetEligibleCodeFromResponse(
88175
signal?: AbortSignal,
89176
): Promise<CodexResetEligibleExhaustionCode | undefined> {
90177
try {
91-
const body = await readBoundedResponseBody(response.clone(), { signal });
178+
const body = await readBoundedResponseBody(response.clone(), { signal, fatalUtf8: true });
92179
if (!body.displaySafe || body.truncated || !body.text.trim()) return undefined;
93-
return structuredResetEligibleCode(JSON.parse(body.text) as unknown);
180+
const payload = JSON.parse(body.text) as unknown;
181+
// JSON.parse silently keeps the last duplicate key, making contradictory
182+
// payloads order-dependent. Reject any duplicate at any object depth.
183+
if (hasDuplicateJsonObjectKeys(body.text)) return undefined;
184+
return structuredResetEligibleCode(payload);
94185
} catch {
95186
// Classification must fail closed. A malformed, oversized, consumed, or
96187
// cancelled body cannot authorize an irreversible reset-credit operation.

‎src/lib/bounded-body.ts‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ export const BOUNDED_BODY_TIMEOUT_MS = 5_000;
77
export interface BoundedBodyOptions {
88
/** Abort the read with this signal. Its reason is rethrown by identity. */
99
signal?: AbortSignal;
10+
/** Reject malformed UTF-8 instead of replacing it. Defaults to false. */
11+
fatalUtf8?: boolean;
1012
/**
1113
* Byte ceiling for retained body data. Defaults to BOUNDED_BODY_MAX_BYTES (64 KiB),
1214
* which suits error bodies; callers materializing whole success payloads (e.g. a
@@ -75,8 +77,8 @@ function cancelWithoutWaiting(reader: ReadableStreamDefaultReader<Uint8Array>, r
7577
}
7678
}
7779

78-
function decodeUtf8(chunks: readonly Uint8Array[]): string {
79-
const decoder = new TextDecoder();
80+
function decodeUtf8(chunks: readonly Uint8Array[], fatal: boolean): string {
81+
const decoder = new TextDecoder("utf-8", { fatal });
8082
let text = "";
8183
for (const chunk of chunks) text += decoder.decode(chunk, { stream: true });
8284
// Flush an incomplete trailing UTF-8 sequence deterministically.
@@ -158,7 +160,7 @@ export async function readBoundedResponseBody(
158160
"TimeoutError",
159161
);
160162
return {
161-
text: decodeUtf8([retained.subarray(0, retainedBytes)]),
163+
text: decodeUtf8([retained.subarray(0, retainedBytes)], options.fatalUtf8 === true),
162164
truncated: true,
163165
timedOut: true,
164166
totalTimedOut: outcome === TOTAL_TIMEOUT,
@@ -171,7 +173,7 @@ export async function readBoundedResponseBody(
171173
const { value, done } = outcome as ReadableStreamReadResult<Uint8Array>;
172174
if (done) {
173175
return {
174-
text: decodeUtf8([retained.subarray(0, retainedBytes)]),
176+
text: decodeUtf8([retained.subarray(0, retainedBytes)], options.fatalUtf8 === true),
175177
truncated: false,
176178
timedOut: false,
177179
totalTimedOut: false,

‎tests/codex-quota-rejection.test.ts‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,24 @@ describe("Codex pre-stream quota rejection classification", () => {
8888
expect(result).not.toHaveProperty("semanticCode");
8989
});
9090

91+
test.each([
92+
["root", '{"code":"rate_limit_error","code":"usage_limit_exceeded"}'],
93+
["nested", '{"error":{"code":"rate_limit_error","code":"usage_limit_exceeded"}}'],
94+
] as const)("fails closed for duplicate keys in a %s object", async (_case, body) => {
95+
const response = new Response(body, {
96+
status: 429,
97+
headers: { "content-type": "application/json" },
98+
});
99+
const result = await classifyCodexPreStreamRejection(response);
100+
expect(result).toMatchObject({
101+
kind: "generic-rate-limit",
102+
alternateRetryEligible: true,
103+
resetCreditEligible: false,
104+
});
105+
expect(result).not.toHaveProperty("semanticCode");
106+
expect(await response.text()).toBe(body);
107+
});
108+
91109
test.each([
92110
["nested unknown code and eligible type", {
93111
error: { code: "unknown", type: "insufficient_quota" },
@@ -139,6 +157,31 @@ describe("Codex pre-stream quota rejection classification", () => {
139157
});
140158
});
141159

160+
test("fails closed when malformed UTF-8 would otherwise be replaced", async () => {
161+
const prefix = new TextEncoder().encode(
162+
'{"error":{"code":"usage_limit_exceeded","message":"',
163+
);
164+
const suffix = new TextEncoder().encode('"}}');
165+
const bytes = new Uint8Array(prefix.length + 1 + suffix.length);
166+
bytes.set(prefix);
167+
bytes[prefix.length] = 0xff;
168+
bytes.set(suffix, prefix.length + 1);
169+
const response = new Response(bytes, {
170+
status: 429,
171+
headers: { "content-type": "application/json" },
172+
});
173+
174+
const result = await classifyCodexPreStreamRejection(response);
175+
176+
expect(result).toMatchObject({
177+
kind: "generic-rate-limit",
178+
alternateRetryEligible: true,
179+
resetCreditEligible: false,
180+
});
181+
expect(result).not.toHaveProperty("semanticCode");
182+
expect(new Uint8Array(await response.arrayBuffer())).toEqual(bytes);
183+
});
184+
142185
test("fails closed for malformed JSON while preserving broad 429 failover", async () => {
143186
const response = new Response('{"error":', {
144187
status: 429,

0 commit comments

Comments
 (0)