|
2 | 2 | // in packages/opencode/src/plugin/codex.ts — the ACTIVE plugin wired |
3 | 3 | // via plugin/index.ts. |
4 | 4 | // |
5 | | -// Filed as issue #1132: GPT 5.6 was released but the allowlist stopped |
6 | | -// at 5.4, so users on ChatGPT Pro/Plus (Codex tier) couldn't pick it in |
7 | | -// the model picker even though the underlying models.dev catalog had it. |
| 5 | +// The allowlist is a verified truth table, not a guess. Every id asserted |
| 6 | +// below was probed against the live backend |
| 7 | +// (POST https://chatgpt.com/backend-api/codex/responses) on a ChatGPT Pro |
| 8 | +// credential, using our own `originator: altimate` client identity over plain |
| 9 | +// HTTP. Accepted ids returned HTTP 200; rejected ids returned |
| 10 | +// 400 {"detail":"The '<id>' model is not supported when using Codex with a |
| 11 | +// ChatGPT account."} |
| 12 | +// |
| 13 | +// Two directions of breakage this file guards: |
| 14 | +// * false positives — an id offered in the picker that 400s at request time |
| 15 | +// (previously gpt-5.2, gpt-5.6, gpt-5.3-codex) |
| 16 | +// * false negatives — a working subscription model hidden from the picker |
| 17 | +// (previously the gpt-5.6 sol/luna/terra variants, i.e. the current |
| 18 | +// flagship models) |
| 19 | +// |
| 20 | +// Supersedes the narrower framing of issue #1132, which added plain gpt-5.6 to |
| 21 | +// the allowlist on catalog presence alone. The backend rejects that id; only |
| 22 | +// its sol/luna/terra variants are actually served on the subscription. |
8 | 23 | // |
9 | 24 | // Sibling file plugin/openai/codex.ts is an in-progress refactor of the |
10 | 25 | // same plugin, currently NOT wired via plugin/index.ts, and NOT covered |
|
17 | 32 | import { describe, expect, test } from "bun:test" |
18 | 33 | import { OAUTH_ALLOWED_MODELS, shouldAllowOAuthModel } from "../../src/plugin/codex" |
19 | 34 |
|
20 | | -describe("OAUTH_ALLOWED_MODELS — subscription model picker regression barrier", () => { |
21 | | - test("issue #1132: gpt-5.6 is present", () => { |
22 | | - // If this fails again, someone dropped gpt-5.6 from the non-codex |
23 | | - // allowlist without moving forward to a newer generation — rejecting |
24 | | - // a shipped OpenAI model users have subscription access to. |
25 | | - expect(OAUTH_ALLOWED_MODELS.has("gpt-5.6")).toBe(true) |
26 | | - }) |
| 35 | +/** Verified HTTP 200 on a ChatGPT Pro subscription credential. */ |
| 36 | +const VERIFIED_ACCEPTED = [ |
| 37 | + "gpt-5.3-codex-spark", |
| 38 | + "gpt-5.4", |
| 39 | + "gpt-5.4-mini", |
| 40 | + "gpt-5.5", |
| 41 | + "gpt-5.6-luna", |
| 42 | + "gpt-5.6-sol", |
| 43 | + "gpt-5.6-terra", |
| 44 | +] |
27 | 45 |
|
28 | | - test("gpt-5.5 is present (added alongside 5.6 for parity)", () => { |
29 | | - expect(OAUTH_ALLOWED_MODELS.has("gpt-5.5")).toBe(true) |
30 | | - }) |
| 46 | +/** Verified HTTP 400 "not supported when using Codex with a ChatGPT account". */ |
| 47 | +const VERIFIED_REJECTED = [ |
| 48 | + "gpt-5", |
| 49 | + "gpt-5.1", |
| 50 | + "gpt-5.2", |
| 51 | + "gpt-5.2-pro", |
| 52 | + "gpt-5.3-chat-latest", |
| 53 | + "gpt-5.3-codex", |
| 54 | + "gpt-5.4-nano", |
| 55 | + "gpt-5.4-pro", |
| 56 | + "gpt-5.5-pro", |
| 57 | + "gpt-5.6", |
| 58 | +] |
31 | 59 |
|
32 | | - test("prior non-codex generations stay allowlisted (no accidental removal)", () => { |
33 | | - for (const id of ["gpt-5.2", "gpt-5.4", "gpt-5.4-mini"]) { |
| 60 | +describe("OAUTH_ALLOWED_MODELS — verified subscription truth table", () => { |
| 61 | + test("every id verified as accepted is allowlisted", () => { |
| 62 | + // False negatives hide working models from the picker. The sol/luna/terra |
| 63 | + // variants are the current flagship subscription models — a previous |
| 64 | + // revision excluded them on the untested assumption that they were |
| 65 | + // API-tier-only, which cost users access to models they already pay for. |
| 66 | + for (const id of VERIFIED_ACCEPTED) { |
34 | 67 | expect(OAUTH_ALLOWED_MODELS.has(id)).toBe(true) |
35 | 68 | } |
36 | 69 | }) |
37 | 70 |
|
38 | | - test("codex-tagged variants are NOT in the non-codex set (they're auto-allowed instead)", () => { |
39 | | - // The non-codex set is deliberately minimal — every codex-tagged id |
40 | | - // is auto-allowed by shouldAllowOAuthModel's `includes("codex")` check |
41 | | - // below, so listing them here would be redundant + a maintenance trap. |
42 | | - for (const id of ["gpt-5.1-codex", "gpt-5.2-codex", "gpt-5.3-codex"]) { |
43 | | - expect(OAUTH_ALLOWED_MODELS.has(id)).toBe(false) |
44 | | - } |
45 | | - }) |
46 | | - |
47 | | - test("allowlist does NOT include API-tier-only variants (defensive)", () => { |
48 | | - // Pro / luna / sol / terra variants ship on models.dev but aren't |
49 | | - // confirmed available on the ChatGPT-subscription (Codex) tier — |
50 | | - // showing them in the picker would surface a request-time failure. |
51 | | - // If OpenAI extends subscription coverage to them, add them here |
52 | | - // deliberately (with a link to the announcement). |
53 | | - for (const id of [ |
54 | | - "gpt-5.4-pro", |
55 | | - "gpt-5.5-pro", |
56 | | - "gpt-5.6-luna", |
57 | | - "gpt-5.6-sol", |
58 | | - "gpt-5.6-terra", |
59 | | - ]) { |
| 71 | + test("every id verified as rejected stays out of the allowlist", () => { |
| 72 | + // False positives are worse than a missing model: the id shows up in the |
| 73 | + // picker, the user selects it, and the request dies with an opaque 400. |
| 74 | + for (const id of VERIFIED_REJECTED) { |
60 | 75 | expect(OAUTH_ALLOWED_MODELS.has(id)).toBe(false) |
61 | 76 | } |
62 | 77 | }) |
63 | 78 |
|
64 | | - test("allowlist size never regresses below current baseline", () => { |
65 | | - // Trip-wire: if someone truncates the allowlist by mistake (or in |
66 | | - // a bad rebase), the count drops and this test catches it before |
67 | | - // shipping. Bump when a real new addition lands. |
68 | | - expect(OAUTH_ALLOWED_MODELS.size).toBeGreaterThanOrEqual(5) |
| 79 | + test("the allowlist contains nothing beyond the verified-accepted set", () => { |
| 80 | + // Trip-wire against speculative additions. To add an id here, probe it |
| 81 | + // against the live endpoint first and land it in VERIFIED_ACCEPTED too. |
| 82 | + expect([...OAUTH_ALLOWED_MODELS].sort()).toEqual([...VERIFIED_ACCEPTED].sort()) |
69 | 83 | }) |
70 | 84 | }) |
71 | 85 |
|
72 | 86 | describe("shouldAllowOAuthModel — behavior of the filter itself", () => { |
73 | 87 | // Behavior-level coverage: even if a refactor stops passing |
74 | 88 | // OAUTH_ALLOWED_MODELS through, the filter function is what the |
75 | 89 | // loader actually calls, so this catches breakage the constant-only |
76 | | - // tests above would miss. (cubic P3 catch.) |
| 90 | + // tests above would miss. |
77 | 91 |
|
78 | | - test("allowlist members pass (spot-check each generation)", () => { |
79 | | - for (const id of ["gpt-5.2", "gpt-5.4", "gpt-5.4-mini", "gpt-5.5", "gpt-5.6"]) { |
| 92 | + test("verified-accepted ids pass", () => { |
| 93 | + for (const id of VERIFIED_ACCEPTED) { |
80 | 94 | expect(shouldAllowOAuthModel(id)).toBe(true) |
81 | 95 | } |
82 | 96 | }) |
83 | 97 |
|
84 | | - test("codex-tagged ids pass regardless of exact allowlist membership", () => { |
85 | | - // Any id containing "codex" auto-passes — covers gpt-5.1-codex, |
86 | | - // gpt-5.3-codex-spark, plus any future codex variant OpenAI ships. |
87 | | - for (const id of [ |
88 | | - "gpt-5.1-codex", |
89 | | - "gpt-5.1-codex-max", |
90 | | - "gpt-5.1-codex-mini", |
91 | | - "gpt-5.2-codex", |
92 | | - "gpt-5.3-codex", |
93 | | - "gpt-5.3-codex-spark", |
94 | | - "gpt-5.3-codex-xhigh", |
95 | | - "codex-hypothetical-future-name", |
96 | | - ]) { |
97 | | - expect(shouldAllowOAuthModel(id)).toBe(true) |
| 98 | + test("verified-rejected ids are filtered out", () => { |
| 99 | + for (const id of VERIFIED_REJECTED) { |
| 100 | + expect(shouldAllowOAuthModel(id)).toBe(false) |
98 | 101 | } |
99 | 102 | }) |
100 | 103 |
|
101 | | - test("API-tier-only variants are rejected (the whole point of the filter)", () => { |
102 | | - // These are the models the previous parseFloat > 5.4 fallback in |
103 | | - // plugin/openai/codex.ts was incorrectly admitting; the shared |
104 | | - // filter must reject them so the picker stays honest about what |
105 | | - // the subscription actually accepts. |
106 | | - for (const id of [ |
107 | | - "gpt-5.4-pro", |
108 | | - "gpt-5.4-nano", |
109 | | - "gpt-5.5-pro", |
110 | | - "gpt-5.6-luna", |
111 | | - "gpt-5.6-sol", |
112 | | - "gpt-5.6-terra", |
113 | | - ]) { |
114 | | - expect(shouldAllowOAuthModel(id)).toBe(false) |
115 | | - } |
| 104 | + test("a 'codex' substring does not grant access on its own", () => { |
| 105 | + // Regression barrier for the removed `modelId.includes("codex")` |
| 106 | + // auto-allow. The backend accepts gpt-5.3-codex-spark but rejects |
| 107 | + // gpt-5.3-codex, so no substring rule can express the real policy — |
| 108 | + // reintroducing one puts broken ids back in the picker. |
| 109 | + expect(shouldAllowOAuthModel("gpt-5.3-codex")).toBe(false) |
| 110 | + expect(shouldAllowOAuthModel("gpt-5.1-codex")).toBe(false) |
| 111 | + expect(shouldAllowOAuthModel("gpt-5.1-codex-max")).toBe(false) |
| 112 | + expect(shouldAllowOAuthModel("codex-hypothetical-future-name")).toBe(false) |
| 113 | + // ...while the one codex id that IS accepted still passes, by exact match. |
| 114 | + expect(shouldAllowOAuthModel("gpt-5.3-codex-spark")).toBe(true) |
116 | 115 | }) |
117 | 116 |
|
118 | 117 | test("completely unrelated ids are rejected", () => { |
119 | | - for (const id of [ |
120 | | - "claude-3.5-sonnet", |
121 | | - "gemini-2.5-pro", |
122 | | - "gpt-4o", |
123 | | - "gpt-4-turbo", |
124 | | - "", |
125 | | - ]) { |
| 118 | + for (const id of ["claude-3.5-sonnet", "gemini-2.5-pro", "gpt-4o", "gpt-4-turbo", ""]) { |
126 | 119 | expect(shouldAllowOAuthModel(id)).toBe(false) |
127 | 120 | } |
128 | 121 | }) |
|
0 commit comments