Skip to content

Commit 7b3bb7a

Browse files
Fix non-ASCII Codex turn metadata headers (#8040)
* Fix non-ASCII Codex turn metadata headers * docs: document ASCII-safe JSON contract for rewritten turn metadata * fix(codex): escape DEL in turn metadata header * docs(structure): record turn metadata header escaping * fix(codex): bound turn metadata by canonical size so re-isolation is idempotent * fix(codex): canonicalize matching turn metadata --------- Co-authored-by: JUN <[email protected]>
1 parent c0221fe commit 7b3bb7a

4 files changed

Lines changed: 139 additions & 8 deletions

File tree

‎src/codex/forward-transport-headers.ts‎

Lines changed: 35 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -20,24 +20,53 @@ export const FORWARD_HEADERS = [
2020
const MAX_ROUTING_MODEL_BYTES = 256;
2121
const MAX_ROUTING_TIER_BYTES = 64;
2222
const MAX_TURN_METADATA_BYTES = 32 * 1024;
23+
// Each escaped code unit (U+007F+) becomes 6 ASCII bytes: at most 3x the UTF-8 size of
24+
// non-ASCII text (DEL alone grows 6x), so 96 KiB holds any DEL-free canonical value.
25+
const MAX_ESCAPED_TURN_METADATA_BYTES = 3 * MAX_TURN_METADATA_BYTES;
2326

2427
/** Compatibility export: caller values never contribute to the selected account identity. */
2528
export function deriveCodexAccountInstallationId(chatgptAccountId: string, _callerInstallationId: string): string {
2629
return codexInstallationIdForAccount(chatgptAccountId);
2730
}
2831

32+
/**
33+
* Rewrites the `installation_id` inside Codex turn metadata while preserving all other fields.
34+
*
35+
* Structure contract: JSON with a nonblank string installation_id is returned as printable ASCII — every code unit outside
36+
* U+0020–U+007E is escaped (JSON.stringify already escapes U+0000–U+001F, with short forms
37+
* such as `\\n`; U+007F and above become `\\uXXXX` here). Both consumers of rewritten turn metadata (the
38+
* `x-codex-turn-metadata` HTTP header and the embedded `client_metadata` value)
39+
* receive this ASCII-safe form, so header setters never see literal Unicode.
40+
*
41+
* The 32 KiB bound applies to the canonical (unescaped UTF-8) rewritten JSON, and the escaped
42+
* result is capped at 96 KiB. Feeding an accepted result back in is therefore a no-op: it
43+
* parses to the same canonical JSON, so repeated isolation passes cannot refuse it.
44+
* Invalid JSON, non-objects, and objects without a nonblank string installation_id pass
45+
* through unchanged up to 32 KiB; their original bytes are not guaranteed header-safe.
46+
*/
2947
export function rewriteTurnMetadataInstallationId(rawMetadata: string, chatgptAccountId: string): string {
3048
if (!chatgptAccountId.trim()) throw new CodexInstallationIdUnavailableError();
3149
// A size bound must refuse the Pool send, never bypass identity projection.
32-
if (Buffer.byteLength(rawMetadata, "utf8") > MAX_TURN_METADATA_BYTES) throw new CodexInstallationIdUnavailableError();
50+
const rawBytes = Buffer.byteLength(rawMetadata, "utf8");
51+
if (rawBytes > MAX_ESCAPED_TURN_METADATA_BYTES) throw new CodexInstallationIdUnavailableError();
52+
// Values returned unchanged keep the raw bound; only rewritten JSON is measured canonically.
53+
const passThrough = () => {
54+
if (rawBytes > MAX_TURN_METADATA_BYTES) throw new CodexInstallationIdUnavailableError();
55+
return rawMetadata;
56+
};
3357
let parsed: unknown;
34-
try { parsed = JSON.parse(rawMetadata); } catch { return rawMetadata; }
35-
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return rawMetadata;
58+
try { parsed = JSON.parse(rawMetadata); } catch { return passThrough(); }
59+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return passThrough();
3660
const record = parsed as Record<string, unknown>;
37-
if (typeof record.installation_id !== "string" || !record.installation_id.trim()) return rawMetadata;
61+
if (typeof record.installation_id !== "string" || !record.installation_id.trim()) return passThrough();
3862
const nextId = deriveCodexAccountInstallationId(chatgptAccountId, record.installation_id);
39-
if (nextId === record.installation_id) return rawMetadata;
40-
return JSON.stringify({ ...record, installation_id: nextId });
63+
const canonical = JSON.stringify({ ...record, installation_id: nextId });
64+
if (Buffer.byteLength(canonical, "utf8") > MAX_TURN_METADATA_BYTES) throw new CodexInstallationIdUnavailableError();
65+
// Turn metadata is also sent as an HTTP header; DEL and non-ASCII are invalid field-value bytes.
66+
const escaped = canonical
67+
.replace(/[\u007f-\uffff]/g, char => `\\u${char.charCodeAt(0).toString(16).padStart(4, "0")}`);
68+
if (escaped.length > MAX_ESCAPED_TURN_METADATA_BYTES) throw new CodexInstallationIdUnavailableError();
69+
return escaped;
4170
}
4271

4372
export function isolateCodexPoolAccountHeaders(headers: Headers, chatgptAccountId: string): void {

‎structure/providers/codex-installation-identity.md‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,19 @@ while empty installation and turn-metadata headers retain their existence gates.
2424
`src/codex/forward-transport-headers.ts` also isolates a present installation field inside native
2525
turn metadata and strips caller attestations from substituted credentials. Its selected-header merge
2626
prevents removed identity fields from reappearing through provider defaults.
27-
Pool turn metadata is bounded to 32 KiB of UTF-8. Larger values refuse the send with the same
27+
For an object with a nonblank string `installation_id`, including one already matching the selected
28+
identity, turn-metadata JSON is canonicalized and escapes every code unit outside
29+
printable ASCII: `JSON.stringify` escapes U+0000–U+001F (with short forms such as `\n` and
30+
`\t` where JSON defines them, `\uXXXX` otherwise), and U+007F and above become `\uXXXX`. The
31+
header stays a valid HTTP field value and JSON-decodes to the same metadata; the client-metadata
32+
copy uses the same form. Pool turn metadata is bounded to 32 KiB of canonical UTF-8, measured on
33+
the unescaped rewritten JSON, and the escaped output is capped at 96 KiB (escaping grows
34+
non-ASCII text at most threefold; DEL-heavy values past the cap are refused). Raw input above
35+
96 KiB is refused before parsing. Invalid JSON, non-objects, and objects without a nonblank string
36+
`installation_id` pass through unchanged up to 32 KiB raw and are not guaranteed header-safe.
37+
Because the bound is on canonical JSON, an accepted escaped value re-enters unchanged, so the
38+
Responses path can isolate it during auth materialization and again in the policy recheck.
39+
Larger values refuse the send with the same
2840
sanitized identity error instead of passing unprojected bytes through; main and main-pool
2941
retain their original values. Routed compact handoffs preserve present empty installation and
3042
turn-metadata headers, so the final selected account applies the same rule as a direct send.

‎tests/codex-integration/codex-metadata-integrity.test.ts‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,9 @@ import {
55
deriveCodexAccountInstallationId,
66
isolateCodexPoolAccountHeaders,
77
isolateCodexPoolClientMetadata,
8+
rewriteTurnMetadataInstallationId,
89
} from "../../src/codex/forward-transport-headers";
10+
import { CodexInstallationIdUnavailableError } from "../../src/codex/installation-id";
911
import type { OcxParsedRequest, OcxProviderConfig } from "../../src/types";
1012
import { withTestTranslatorBudget } from "../helpers/translator-budget";
1113

@@ -226,6 +228,64 @@ describe("Codex metadata integrity", () => {
226228
isolateCodexPoolAccountHeaders(copyHeaders, accountA.chatgptAccountId);
227229
expect(copyHeaders.get("x-codex-installation-id")).toBe(idA);
228230
});
231+
test("pooled turn metadata preserves non-ASCII values in header-safe JSON", () => {
232+
const raw = String.raw`{"installation_id":"caller-installation","workspaces":["/tmp/\ubcd1\uc6d0/\ud83d\ude80"],"extra":{"note":"\ud55c\uae00 \ud83d\ude00"}}`;
233+
const incoming = new Headers({ "x-codex-turn-metadata": raw });
234+
const headers = headersForCodexAuthContext(incoming, poolAuthContext);
235+
const rewritten = headers.get("x-codex-turn-metadata")!;
236+
expect(rewritten).toMatch(/^[\x00-\x7f]*$/);
237+
expect(JSON.parse(rewritten)).toEqual({
238+
...JSON.parse(raw),
239+
installation_id: deriveCodexAccountInstallationId(accountA.chatgptAccountId, "caller-installation"),
240+
});
241+
const body = { client_metadata: { "x-codex-turn-metadata": raw } };
242+
expect((isolateCodexPoolClientMetadata(body, accountA.chatgptAccountId) as typeof body)
243+
.client_metadata["x-codex-turn-metadata"]).toBe(rewritten);
244+
isolateCodexPoolAccountHeaders(headers, accountA.chatgptAccountId);
245+
expect(headers.get("x-codex-turn-metadata")).toBe(rewritten);
246+
});
247+
test("pooled turn metadata escapes DEL so the header stays a valid field value", () => {
248+
const raw = String.raw`{"installation_id":"caller-installation","workspaces":["/tmp/a\u007fb/\ubcd1\u0001"]}`;
249+
const headers = headersForCodexAuthContext(new Headers({ "x-codex-turn-metadata": raw }), poolAuthContext);
250+
const rewritten = headers.get("x-codex-turn-metadata")!;
251+
expect(rewritten).toMatch(/^[\x20-\x7e\t]*$/);
252+
expect(JSON.parse(rewritten).workspaces).toEqual(["/tmp/a\u007fb/\ubcd1\u0001"]);
253+
expect(JSON.parse(rewritten).installation_id)
254+
.toBe(deriveCodexAccountInstallationId(accountA.chatgptAccountId, "caller-installation"));
255+
});
256+
test("escaped turn metadata is accepted unchanged on re-entry and capped at 96 KiB", () => {
257+
const account = accountA.chatgptAccountId;
258+
const raw = JSON.stringify({ installation_id: "caller-installation", pad: "\u00e9".repeat(6000) });
259+
const once = rewriteTurnMetadataInstallationId(raw, account);
260+
expect(once.length).toBeGreaterThan(32 * 1024);
261+
expect(rewriteTurnMetadataInstallationId(once, account)).toBe(once);
262+
const headers = headersForCodexAuthContext(new Headers({ "x-codex-turn-metadata": raw }), poolAuthContext);
263+
expect(headersForCodexAuthContext(headers, poolAuthContext).get("x-codex-turn-metadata")).toBe(once);
264+
// DEL expands 6x: 20,000 of them fit the 32 KiB canonical bound but exceed the escaped cap.
265+
const del = JSON.stringify({ installation_id: "caller-installation", pad: "\u007f".repeat(20_000) });
266+
expect(() => rewriteTurnMetadataInstallationId(del, account)).toThrow(CodexInstallationIdUnavailableError);
267+
// Non-JSON values pass through unchanged, so they keep the raw 32 KiB bound.
268+
expect(() => rewriteTurnMetadataInstallationId("x".repeat(32 * 1024 + 1), account)).toThrow(CodexInstallationIdUnavailableError);
269+
});
270+
for (const value of ["漢字", "a\u007fb"]) test(`equal installation IDs escape ${JSON.stringify(value)}`, () => {
271+
const account = accountA.chatgptAccountId;
272+
const installation_id = deriveCodexAccountInstallationId(account, "caller");
273+
const raw = JSON.stringify({ installation_id, value });
274+
const once = rewriteTurnMetadataInstallationId(raw, account);
275+
expect(once).toMatch(/^[\x20-\x7e]*$/);
276+
expect(JSON.parse(once)).toEqual({ installation_id, value });
277+
expect(rewriteTurnMetadataInstallationId(once, account)).toBe(once);
278+
});
279+
test("equal installation IDs use canonical and escaped size bounds", () => {
280+
const account = accountA.chatgptAccountId;
281+
const installation_id = deriveCodexAccountInstallationId(account, "caller");
282+
const padded = ` ${" ".repeat(90_000)}${JSON.stringify({ installation_id })}`;
283+
expect(rewriteTurnMetadataInstallationId(padded, account)).toBe(JSON.stringify({ installation_id }));
284+
const oversized = JSON.stringify({ installation_id, pad: "a".repeat(32 * 1024) });
285+
expect(() => rewriteTurnMetadataInstallationId(oversized, account)).toThrow(CodexInstallationIdUnavailableError);
286+
const del = JSON.stringify({ installation_id, pad: "\u007f".repeat(20_000) });
287+
expect(() => rewriteTurnMetadataInstallationId(del, account)).toThrow(CodexInstallationIdUnavailableError);
288+
});
229289
test("Responses preserves caller User-Agent as a fallback in key and forward modes", async () => {
230290
for (const provider of [
231291
{

‎tests/responses/codex-installation-turn-limit.test.ts‎

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ import { afterEach, beforeEach, expect, test } from "bun:test";
22
import { handleResponses, handleResponsesCompact } from "../../src/server/responses";
33
import { handleSearch } from "../../src/server/search";
44
import { acquireOwnedSpendHome } from "../helpers/owned-spend-home";
5-
import { install, installationLiveFixture, liveConfig, turn, turnMetadataBytes } from "../helpers/codex-installation-live";
5+
import { callerId, install, installationLiveFixture, liveConfig, turn, turnMetadataBytes } from "../helpers/codex-installation-live";
66

77
const fixture = installationLiveFixture();
88
const originalFetch = globalThis.fetch;
@@ -16,6 +16,36 @@ beforeEach(() => {
1616
}) as typeof fetch;
1717
});
1818
afterEach(() => { globalThis.fetch = originalFetch; release(); });
19+
// Non-ASCII pad of exactly `bytes` canonical UTF-8; escaping it expands the wire form ~3x.
20+
function unicodeTurnMetadataBytes(bytes: number) {
21+
const room = bytes - Buffer.byteLength(JSON.stringify({ installation_id: callerId, pad: "" }));
22+
return JSON.stringify({ installation_id: callerId, pad: "a".repeat(room % 2) + "\u00e9".repeat(Math.floor(room / 2)) });
23+
}
24+
for (const surface of ["Responses", "compact", "search"] as const)
25+
for (const header of [false, true]) for (const bytes of [12038, 32768, 32769]) {
26+
test(`${surface} ${header ? "header" : "body-only"} non-ASCII turn metadata ${bytes} canonical bytes`, async () => {
27+
const raw = unicodeTurnMetadataBytes(bytes); expect(Buffer.byteLength(raw)).toBe(bytes);
28+
const config = liveConfig(); config.providers.openai!.upstreamWebsocket = false;
29+
const headers = new Headers({ "content-type": "application/json" }); if (header) headers.set(turn, raw);
30+
const req = new Request(`http://localhost/v1/${surface === "compact" ? "responses/compact" : surface === "search" ? "alpha/search" : "responses"}`, {
31+
method: "POST", headers, body: JSON.stringify({ model: "gpt-5.5", input: [], stream: false,
32+
...(header ? {} : { client_metadata: { [turn]: raw } }) }),
33+
});
34+
const handler = surface === "compact" ? handleResponsesCompact : surface === "search" ? handleSearch : handleResponses;
35+
const response = await handler(req, config, { model: "", provider: "" });
36+
expect(response.status).toBe(bytes > 32768 ? 503 : 200);
37+
await response.text();
38+
expect(sent).toHaveLength(bytes > 32768 ? 0 : 1);
39+
if (bytes <= 32768) {
40+
const wire = sent[0]!;
41+
const metadata: string = header ? wire.headers.get(turn)! : (await wire.json()).client_metadata[turn];
42+
// The escaped form exceeds 32 KiB; repeated isolation passes must still accept it.
43+
expect(metadata).toMatch(/^[\x20-\x7e]*$/);
44+
expect(metadata.length).toBeGreaterThan(Buffer.byteLength(raw));
45+
expect(JSON.parse(metadata)).toEqual({ installation_id: fixture.expected(), pad: JSON.parse(raw).pad });
46+
}
47+
});
48+
}
1949
for (const surface of ["Responses", "compact", "search"] as const)
2050
for (const header of [false, true]) for (const bytes of [32768, 32769]) {
2151
test(`${surface} ${header ? "header" : "body-only"} turn metadata ${bytes} bytes`, async () => {

0 commit comments

Comments
 (0)