Skip to content

Repository navigation

Commit 5331af1

Browse files
ryoppippiryoppippi
committed
fix(codex): generate review fallback models
Replace the hand-written codex-auto-review release table in the parser with an embedded snapshot generated from the pinned models.dev input. Extend the models.dev snapshot generator to emit Codex auto-review fallback metadata alongside the existing pricing snapshot, and wire the just recipe to refresh both generated files. Keep runtime lookup fast by parsing the tiny embedded snapshot once through LazyLock, while preserving the existing date-aware fallback behavior and malformed-date conservative fallback.
1 parent 8f0937e commit 5331af1

7 files changed

Lines changed: 164 additions & 33 deletions

File tree

‎docs/guide/codex/index.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ These views support `--json`, `--compact`, `--offline`, and `--speed auto|standa
6060
| `CODEX_HOME` | Override the root directory, or comma-separated directories, containing Codex homes or saved `codex exec --json` JSONL files |
6161
| `LOG_LEVEL` | Adjust log verbosity (0 silent … 5 trace) |
6262

63-
When Codex emits a model alias, the CLI automatically resolves it through the LiteLLM pricing data when possible. For `codex-auto-review`, the Codex parser maps the label to the newest known Codex/OpenAI model available on the log date before pricing uses the resolved model name. No manual override is needed.
63+
When Codex emits a model alias, the CLI automatically resolves it through the LiteLLM pricing data when possible. For `codex-auto-review`, the Codex parser maps the label to the newest known Codex/OpenAI model available on the log date using a pinned models.dev snapshot before pricing uses the resolved model name. No manual override is needed.
6464

6565
## Speed Pricing
6666

‎justfile‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,11 +54,13 @@ update-litellm-pricing:
5454
nix flake update litellm
5555
just check
5656

57-
# Regenerate the committed models.dev pricing snapshot from the pinned input
57+
# Regenerate committed models.dev snapshots from the pinned input
5858
gen-models-dev-pricing:
59-
cp "$(nix build .#models-dev-pricing --no-link --print-out-paths)" rust/crates/ccusage/src/models-dev-pricing.json
59+
snapshots="$(nix build .#models-dev-pricing --no-link --print-out-paths)"; cp "$snapshots/models-dev-pricing.json" rust/crates/ccusage/src/models-dev-pricing.json; cp "$snapshots/codex-auto-review-fallbacks.json" rust/crates/ccusage/src/adapter/codex/codex-auto-review-fallbacks.json
6060
chmod u+w rust/crates/ccusage/src/models-dev-pricing.json
61+
chmod u+w rust/crates/ccusage/src/adapter/codex/codex-auto-review-fallbacks.json
6162
nix fmt rust/crates/ccusage/src/models-dev-pricing.json
63+
nix fmt rust/crates/ccusage/src/adapter/codex/codex-auto-review-fallbacks.json
6264

6365
# Update the pinned models.dev input, regenerate its pricing snapshot, and validate
6466
update-models-dev-pricing:

‎nix/models-dev-gen.ts‎

Lines changed: 72 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,10 @@
77
* Anthropic-relevant models and the few pricing fields ccusage consumes. The
88
* embedded output is a flat map keyed by runtime model id. The output is
99
* committed to the repository and embedded at build time, so every platform
10-
* ships the identical, pinned data without any build-time network access. Run
11-
* via `just gen-models-dev-pricing` (see `nix/models-dev-pricing.nix`).
10+
* ships the identical, pinned data without any build-time network access. The
11+
* same pinned catalog also generates the Codex auto-review fallback metadata
12+
* used by the Rust parser. Run via `just gen-models-dev-pricing` (see
13+
* `nix/models-dev-pricing.nix`).
1214
*/
1315
import { generateCatalog } from './packages/core/src/generate.ts';
1416
import {
@@ -26,13 +28,22 @@ type Cost = {
2628
cache_write?: number | null;
2729
};
2830
type Model = { id?: string; cost?: Cost; limit?: { context?: number | null } };
31+
type ModelMetadata = {
32+
id?: string;
33+
release_date?: string;
34+
};
2935
type Provider = { models?: Record<string, Model> };
3036
type EmbeddedModel = {
3137
cost: Cost;
3238
limit?: { context: number };
3339
};
40+
type CodexAutoReviewFallback = {
41+
releasedOn: string;
42+
model: string;
43+
};
3444

35-
const { providers } = (await generateCatalog('.')) as {
45+
const { models, providers } = (await generateCatalog('.')) as {
46+
models: Record<string, ModelMetadata>;
3647
providers: Record<string, Provider>;
3748
};
3849

@@ -95,3 +106,61 @@ if (outfile == null || outfile.length === 0) {
95106
}
96107

97108
await Bun.write(outfile, `${JSON.stringify(sortObject(out), null, 2)}\n`);
109+
110+
const codexFallbacksOutfile = process.env.CODEX_AUTO_REVIEW_FALLBACKS_OUTFILE;
111+
if (codexFallbacksOutfile != null && codexFallbacksOutfile.length > 0) {
112+
await Bun.write(
113+
codexFallbacksOutfile,
114+
`${JSON.stringify(generateCodexAutoReviewFallbacks(models), null, 2)}\n`,
115+
);
116+
}
117+
118+
function generateCodexAutoReviewFallbacks(
119+
models: Record<string, ModelMetadata>,
120+
): CodexAutoReviewFallback[] {
121+
const entries = Object.entries(models).filter(([modelId, model]) =>
122+
isCodexAutoReviewFallbackCandidate(modelId, model),
123+
);
124+
const codexDecimalVersions = new Set(
125+
entries
126+
.map(([modelId]) => codexDecimalVersion(openAiModelName(modelId)))
127+
.filter((version): version is string => version != null),
128+
);
129+
130+
return entries
131+
.filter(([modelId]) => {
132+
const version = baseDecimalVersion(openAiModelName(modelId));
133+
return version == null || !codexDecimalVersions.has(version);
134+
})
135+
.map(([modelId, model]) => ({
136+
releasedOn: model.release_date!,
137+
model: openAiModelName(model.id ?? modelId),
138+
}))
139+
.sort((left, right) => right.releasedOn.localeCompare(left.releasedOn));
140+
}
141+
142+
function isCodexAutoReviewFallbackCandidate(modelId: string, model: ModelMetadata): boolean {
143+
if (model.release_date == null || !/^\d{4}-\d{2}-\d{2}$/.test(model.release_date)) {
144+
return false;
145+
}
146+
const modelName = openAiModelName(modelId);
147+
return (
148+
modelName === 'gpt-5' ||
149+
modelName === 'gpt-5-codex' ||
150+
/^gpt-5\.\d+$/.test(modelName) ||
151+
/^gpt-5\.\d+-codex$/.test(modelName)
152+
);
153+
}
154+
155+
function baseDecimalVersion(modelId: string): string | undefined {
156+
return /^gpt-5\.\d+$/.test(modelId) ? modelId : undefined;
157+
}
158+
159+
function codexDecimalVersion(modelId: string): string | undefined {
160+
const match = /^(gpt-5\.\d+)-codex$/.exec(modelId);
161+
return match?.[1];
162+
}
163+
164+
function openAiModelName(modelId: string): string {
165+
return modelId.startsWith('openai/') ? modelId.slice('openai/'.length) : modelId;
166+
}

‎nix/models-dev-pricing.nix‎

Lines changed: 14 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,13 @@
1-
# Reproducibly regenerate the compacted models.dev pricing snapshot from the
2-
# pinned `models-dev` flake input. The upstream repository ships per-model TOML
3-
# sources rather than a prebuilt catalog, so we run their own `generateCatalog`
4-
# routine (via `nix/models-dev-gen.ts`) with Bun. Its only runtime dependencies
5-
# are `remeda` and `zod` (both zero-dependency packages), which we vendor
6-
# directly from the registry using the integrity hashes pinned in upstream
7-
# `bun.lock`.
1+
# Reproducibly regenerate compacted models.dev snapshots from the pinned
2+
# `models-dev` flake input. The upstream repository ships per-model TOML sources
3+
# rather than a prebuilt catalog, so we run their own `generateCatalog` routine
4+
# (via `nix/models-dev-gen.ts`) with Bun. Its only runtime dependencies are
5+
# `remeda` and `zod` (both zero-dependency packages), which we vendor directly
6+
# from the registry using the integrity hashes pinned in upstream `bun.lock`.
87
#
9-
# The build output is copied into the repository (see `just gen-models-dev-pricing`)
10-
# and embedded at build time, so every platform ships identical pinned data
11-
# without build-time network access.
8+
# The build outputs are copied into the repository (see
9+
# `just gen-models-dev-pricing`) and embedded at build time, so every platform
10+
# ships identical pinned data without build-time network access.
1211
{
1312
pkgs,
1413
modelsDevSrc,
@@ -26,7 +25,7 @@ let
2625
hash = "sha512-lY7CDW43ECgW9u1TcT3IoXHflywfVqDYze4waEz812jR/bZ8FHDsl7pFQoSZTz5N+2NqRXs8GBwnAwo3ZNxqhQ==";
2726
};
2827
in
29-
pkgs.runCommand "models-dev-pricing.json"
28+
pkgs.runCommand "models-dev-snapshots"
3029
{
3130
nativeBuildInputs = [ pkgs.bun ];
3231
}
@@ -52,5 +51,8 @@ pkgs.runCommand "models-dev-pricing.json"
5251
cp ${./models-dev-compact.ts} work/models-dev-compact.ts
5352
5453
cd work
55-
OUTFILE="$out" bun run gen.ts
54+
mkdir -p "$out"
55+
OUTFILE="$out/models-dev-pricing.json" \
56+
CODEX_AUTO_REVIEW_FALLBACKS_OUTFILE="$out/codex-auto-review-fallbacks.json" \
57+
bun run gen.ts
5658
''

‎nix/packages.nix‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,9 +27,9 @@ in
2727
;
2828
};
2929
ccusageProgram = pkgs.lib.getExe' ccusage "ccusage";
30-
# Regeneration-only output for the committed models.dev pricing snapshot;
31-
# `just gen-models-dev-pricing` builds this and copies it into the source
32-
# tree. It is not part of the ccusage build, which embeds the committed file.
30+
# Regeneration-only output for committed models.dev snapshots;
31+
# `just gen-models-dev-pricing` builds this and copies them into the source
32+
# tree. It is not part of the ccusage build, which embeds the committed files.
3333
models-dev-pricing = pkgs.callPackage ../nix/models-dev-pricing.nix {
3434
modelsDevSrc = inputs.models-dev;
3535
};

‎rust/crates/ccusage/src/adapter/codex/codex-auto-review-fallbacks.json‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
[
2+
{
3+
"releasedOn": "2026-04-23",
4+
"model": "gpt-5.5"
5+
},
6+
{
7+
"releasedOn": "2026-03-05",
8+
"model": "gpt-5.4"
9+
},
10+
{
11+
"releasedOn": "2026-02-05",
12+
"model": "gpt-5.3-codex"
13+
},
14+
{
15+
"releasedOn": "2025-12-11",
16+
"model": "gpt-5.2-codex"
17+
},
18+
{
19+
"releasedOn": "2025-11-13",
20+
"model": "gpt-5.1-codex"
21+
},
22+
{
23+
"releasedOn": "2025-09-15",
24+
"model": "gpt-5-codex"
25+
},
26+
{
27+
"releasedOn": "2025-08-07",
28+
"model": "gpt-5"
29+
}
30+
]

‎rust/crates/ccusage/src/adapter/codex/parser.rs‎

Lines changed: 40 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ use std::{
77
};
88

99
use memchr::memmem::Finder;
10+
use serde::Deserialize;
1011
use serde_json::Value;
1112

1213
use crate::{CodexRawUsage, CodexTokenUsageEvent, Result, TimestampMs};
@@ -35,16 +36,20 @@ static THREAD_SPAWN_FINDER: LazyLock<Finder<'static>> =
3536
LazyLock::new(|| Finder::new(b"thread_spawn"));
3637

3738
const CODEX_AUTO_REVIEW_MODEL: &str = "codex-auto-review";
38-
// New entries must be inserted in descending release-date order.
39-
const CODEX_AUTO_REVIEW_FALLBACK_MODELS: [(&str, &str); 7] = [
40-
("2026-04-23", "gpt-5.5"),
41-
("2026-03-05", "gpt-5.4"),
42-
("2026-02-05", "gpt-5.3-codex"),
43-
("2025-12-11", "gpt-5.2-codex"),
44-
("2025-11-13", "gpt-5.1-codex"),
45-
("2025-09-15", "gpt-5-codex"),
46-
("2025-08-07", "gpt-5"),
47-
];
39+
const CODEX_AUTO_REVIEW_FALLBACKS_JSON: &str = include_str!("codex-auto-review-fallbacks.json");
40+
41+
static CODEX_AUTO_REVIEW_FALLBACK_MODELS: LazyLock<Vec<CodexAutoReviewFallback<'static>>> =
42+
LazyLock::new(|| {
43+
serde_json::from_str(CODEX_AUTO_REVIEW_FALLBACKS_JSON)
44+
.expect("embedded codex-auto-review fallback snapshot must parse")
45+
});
46+
47+
#[derive(Deserialize)]
48+
#[serde(rename_all = "camelCase")]
49+
struct CodexAutoReviewFallback<'a> {
50+
released_on: &'a str,
51+
model: &'a str,
52+
}
4853

4954
#[derive(Clone, Copy)]
5055
enum CodexLineKind {
@@ -517,13 +522,17 @@ fn codex_log_model_fallback(model: &str, timestamp: &str) -> Option<&'static str
517522
return Some("gpt-5");
518523
};
519524
Some(
520-
CODEX_AUTO_REVIEW_FALLBACK_MODELS
525+
codex_auto_review_fallback_models()
521526
.iter()
522-
.find_map(|(released_on, fallback)| (date >= *released_on).then_some(*fallback))
527+
.find_map(|fallback| (date >= fallback.released_on).then_some(fallback.model))
523528
.unwrap_or("gpt-5"),
524529
)
525530
}
526531

532+
fn codex_auto_review_fallback_models() -> &'static [CodexAutoReviewFallback<'static>] {
533+
CODEX_AUTO_REVIEW_FALLBACK_MODELS.as_slice()
534+
}
535+
527536
fn codex_timestamp_date(timestamp: &str) -> Option<&str> {
528537
let date = timestamp.get(..10)?;
529538
let bytes = date.as_bytes();
@@ -951,3 +960,22 @@ fn subtract_codex_raw_usage(
951960
.saturating_sub(previous.map_or(0, |usage| usage.total_tokens)),
952961
}
953962
}
963+
964+
#[cfg(test)]
965+
mod tests {
966+
use super::*;
967+
968+
#[test]
969+
fn loads_codex_auto_review_fallbacks_from_models_dev_snapshot() {
970+
let fallbacks = codex_auto_review_fallback_models();
971+
972+
assert_eq!(fallbacks.len(), 7);
973+
assert_eq!(fallbacks[0].released_on, "2026-04-23");
974+
assert_eq!(fallbacks[0].model, "gpt-5.5");
975+
assert_eq!(fallbacks[6].released_on, "2025-08-07");
976+
assert_eq!(fallbacks[6].model, "gpt-5");
977+
assert!(fallbacks
978+
.windows(2)
979+
.all(|window| window[0].released_on > window[1].released_on));
980+
}
981+
}

0 commit comments

Comments
 (0)