Skip to content

Commit 8ffefa0

Browse files
committed
Merge branch 'main' into fix/audit-m064
The one conflict, in GitVcsDriverCore.test.ts after the git helper, keeps both sides: this branch's Git version check for the reftable cases and main's new gitWithStdin helper.
2 parents 80876e5 + a6b8f71 commit 8ffefa0

572 files changed

Lines changed: 27828 additions & 5219 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/actions/setup-ci-deps/action.yml‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ runs:
3535
- name: Cache Vite+
3636
id: vp_cache
3737
if: steps.vp.outputs.version != ''
38-
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
38+
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
3939
with:
4040
path: ~/.vite-plus
4141
key: vite-plus-home-v2-${{ runner.os }}-${{ runner.arch }}-${{ steps.vp.outputs.version }}-node${{ steps.vp.outputs.node }}-${{ steps.vp.outputs.pnpm }}
@@ -56,7 +56,7 @@ runs:
5656
# partial bytes, so a shorter limit costs a slow download only a retry.
5757
- name: Setup Vite+
5858
if: steps.vp_cache.outputs.cache-hit != 'true'
59-
uses: voidzero-dev/setup-vp@250f29ce396baf5e8f24498e17c0dfdebabc26eb # v1
59+
uses: voidzero-dev/setup-vp@250f29ce396baf5e8f24498e17c0dfdebabc26eb # v1.15.0
6060
env:
6161
VP_DOWNLOAD_TIMEOUT: "120"
6262
with:
@@ -92,7 +92,7 @@ runs:
9292
- name: Restore pnpm store
9393
id: pnpm_cache
9494
if: inputs.install == 'true'
95-
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
95+
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
9696
with:
9797
path: ${{ steps.pnpm_store.outputs.path }}
9898
key: pnpm-${{ steps.pnpm_store.outputs.namespace }}-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('pnpm-lock.yaml', 'pnpm-workspace.yaml', 'package.json') }}
@@ -106,7 +106,7 @@ runs:
106106
- name: Restore lockfile verification
107107
id: pnpm_verified
108108
if: inputs.install == 'true'
109-
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
109+
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
110110
with:
111111
path: ${{ steps.pnpm_store.outputs.verified }}
112112
key: pnpm-verified-v1-${{ hashFiles('pnpm-lock.yaml', 'pnpm-workspace.yaml', 'package.json') }}
@@ -121,7 +121,7 @@ runs:
121121

122122
- name: Save lockfile verification
123123
if: inputs.install == 'true' && steps.pnpm_verified.outputs.cache-hit != 'true'
124-
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
124+
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
125125
with:
126126
path: ${{ steps.pnpm_store.outputs.verified }}
127127
key: ${{ steps.pnpm_verified.outputs.cache-primary-key }}
@@ -138,7 +138,7 @@ runs:
138138
steps.pnpm_cache.outputs.cache-hit != 'true' &&
139139
(steps.pnpm_store.outputs.namespace == 'full' ||
140140
steps.pnpm_cache.outputs.cache-matched-key != format('pnpm-full-v1-{0}-{1}-{2}', runner.os, runner.arch, hashFiles('pnpm-lock.yaml', 'pnpm-workspace.yaml', 'package.json')))
141-
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
141+
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
142142
with:
143143
path: ${{ steps.pnpm_store.outputs.path }}
144144
key: ${{ steps.pnpm_cache.outputs.cache-primary-key }}

‎.github/scripts/thread-transfer-report.cjs‎

Lines changed: 150 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ const path = require("node:path");
33

44
const ARTIFACT_NAME = "thread-transfer-results";
55
const RESULT_FILE = "thread-transfer-result.json";
6+
const SCENARIOS_FILE = "thread-transfer-scenarios.json";
67
const COMMENT_MARKER = "<!-- t3-thread-transfer-report -->";
78
const PROVIDERS = ["codex", "claudeAgent"];
89
const OBSERVED_KEYS = [
@@ -29,6 +30,12 @@ const SCENARIO_KEYS = [
2930
"measuredMcpResultBytes",
3031
];
3132

33+
const SCENARIO_ROW_KEYS = ["phase", "metric", "unit", "observed", "ceiling"];
34+
// These labels come from the PR and reach a comment the bot posts: no markup, links or mentions.
35+
// GitHub still links plain text such as `GH-26`, so they are rendered as code spans.
36+
const SCENARIO_LABEL = /^[A-Za-z0-9][A-Za-z0-9 ,-]{0,79}$/;
37+
const MAX_SCENARIO_ROWS = 100;
38+
3239
function resultShaMarker(sha) {
3340
return `<!-- t3-thread-transfer-result-sha:${sha} -->`;
3441
}
@@ -85,15 +92,63 @@ function validateResult(value) {
8592
return value;
8693
}
8794

88-
function readResult(directory) {
95+
function validateScenarios(value) {
96+
assertExactKeys(value, ["schemaVersion", "providers"], "scenarios");
97+
if (value.schemaVersion !== 1) {
98+
throw new Error("scenarios.schemaVersion must be 1");
99+
}
100+
assertExactKeys(value.providers, PROVIDERS, "scenarios.providers");
101+
for (const provider of PROVIDERS) {
102+
const rows = value.providers[provider];
103+
if (!Array.isArray(rows) || rows.length > MAX_SCENARIO_ROWS) {
104+
throw new Error(
105+
`scenarios.providers.${provider} must list at most ${MAX_SCENARIO_ROWS} rows`,
106+
);
107+
}
108+
const seen = new Set();
109+
for (const [index, row] of rows.entries()) {
110+
const label = `scenarios.providers.${provider}[${index}]`;
111+
assertExactKeys(row, SCENARIO_ROW_KEYS, label);
112+
for (const key of ["phase", "metric"]) {
113+
if (typeof row[key] !== "string" || !SCENARIO_LABEL.test(row[key])) {
114+
throw new Error(`${label}.${key} must be a short plain label`);
115+
}
116+
}
117+
if (row.unit !== "bytes" && row.unit !== "count") {
118+
throw new Error(`${label}.unit must be bytes or count`);
119+
}
120+
assertMetric(row.observed, `${label}.observed`);
121+
assertMetric(row.ceiling, `${label}.ceiling`);
122+
const key = scenarioKey(row);
123+
if (seen.has(key)) throw new Error(`${label} repeats an earlier row`);
124+
seen.add(key);
125+
}
126+
}
127+
return value;
128+
}
129+
130+
function scenarioKey(row) {
131+
return `${row.phase}|${row.metric}`;
132+
}
133+
134+
function readArtifactFile(directory, name, validate) {
89135
if (!directory) return undefined;
90-
const file = path.join(directory, RESULT_FILE);
136+
const file = path.join(directory, name);
91137
if (!fs.existsSync(file)) return undefined;
92138
const stat = fs.lstatSync(file);
93139
if (!stat.isFile() || stat.size > 64 * 1_024) {
94-
throw new Error("thread transfer result must be a regular file smaller than 64 KiB");
140+
throw new Error(`${name} must be a regular file smaller than 64 KiB`);
95141
}
96-
return validateResult(JSON.parse(fs.readFileSync(file, "utf8")));
142+
return validate(JSON.parse(fs.readFileSync(file, "utf8")));
143+
}
144+
145+
function readResult(directory) {
146+
return readArtifactFile(directory, RESULT_FILE, validateResult);
147+
}
148+
149+
// Older runs upload only the result file, so a run without scenarios is still a valid run.
150+
function readScenarios(directory) {
151+
return readArtifactFile(directory, SCENARIOS_FILE, validateScenarios);
97152
}
98153

99154
function formatBytes(bytes) {
@@ -103,7 +158,7 @@ function formatBytes(bytes) {
103158
}
104159

105160
function formatValue(value, kind) {
106-
return kind === "messages" ? value.toLocaleString("en-US") : formatBytes(value);
161+
return kind === "bytes" ? formatBytes(value) : value.toLocaleString("en-US");
107162
}
108163

109164
function formatImpact(current, baseline, kind) {
@@ -136,6 +191,77 @@ const METRICS = [
136191
{ key: "measuredTurnWebSocketMessages", label: "Live turn messages", kind: "messages" },
137192
];
138193

194+
function providerLabel(provider) {
195+
return provider === "codex" ? "Codex" : "Claude";
196+
}
197+
198+
/**
199+
* The scenario rows of a run against the same rows on `main`. Rows that fail or differ from
200+
* `main` are listed in the open; the full table is collapsed below them.
201+
*/
202+
function renderScenarios(current, baseline, ceilingChanges) {
203+
if (!current) return { lines: [], failed: false };
204+
const header = [
205+
"| Provider | Scenario | Metric | Main baseline | This PR | Impact | PR ceiling | |",
206+
"| --- | --- | --- | ---: | ---: | ---: | ---: | --- |",
207+
];
208+
const all = [];
209+
const notable = [];
210+
let failed = false;
211+
let withoutBaseline = 0;
212+
213+
for (const provider of PROVIDERS) {
214+
const baselineRows = new Map(
215+
(baseline?.providers[provider] ?? []).map((row) => [scenarioKey(row), row]),
216+
);
217+
for (const row of current.providers[provider]) {
218+
const before = baselineRows.get(scenarioKey(row));
219+
const beforeObserved = before?.unit === row.unit ? before.observed : undefined;
220+
const pass = row.observed <= row.ceiling;
221+
failed ||= !pass;
222+
if (beforeObserved === undefined) withoutBaseline += 1;
223+
const line = `| ${providerLabel(provider)} | \`${row.phase}\` | \`${row.metric}\` | ${beforeObserved === undefined ? "—" : formatValue(beforeObserved, row.unit)} | ${formatValue(row.observed, row.unit)} | ${formatImpact(row.observed, beforeObserved, row.unit)} | ${formatValue(row.ceiling, row.unit)} | ${pass ? "✅" : "❌"} |`;
224+
all.push(line);
225+
if (!pass || (beforeObserved !== undefined && beforeObserved !== row.observed)) {
226+
notable.push(line);
227+
}
228+
if (beforeObserved !== undefined && before.ceiling !== row.ceiling) {
229+
ceilingChanges.push(
230+
`- ${providerLabel(provider)} \`${row.phase}\` \`${row.metric}\`: ${formatValue(before.ceiling, row.unit)} → ${formatValue(row.ceiling, row.unit)}`,
231+
);
232+
}
233+
}
234+
}
235+
236+
const compared = all.length - withoutBaseline;
237+
const summary = !baseline
238+
? `No \`main\` baseline exists yet for these ${all.length} scenario measurements.`
239+
: [
240+
notable.length > 0
241+
? `${notable.length} of ${all.length} scenario measurements changed or failed.`
242+
: `${compared === all.length ? "All " : ""}${compared} scenario measurements match \`main\`.`,
243+
...(withoutBaseline > 0 ? [`${withoutBaseline} have no \`main\` baseline.`] : []),
244+
].join(" ");
245+
return {
246+
failed,
247+
lines: [
248+
"### Other scenarios",
249+
"",
250+
summary,
251+
...(notable.length > 0 ? ["", ...header, ...notable] : []),
252+
"",
253+
"<details>",
254+
`<summary>All ${all.length} scenario measurements</summary>`,
255+
"",
256+
...header,
257+
...all,
258+
"",
259+
"</details>",
260+
"",
261+
],
262+
};
263+
}
264+
139265
function renderComment(input) {
140266
const current = input.current;
141267
const baseline = input.baseline;
@@ -154,16 +280,22 @@ function renderComment(input) {
154280
const pass = observed <= ceiling;
155281
failed ||= !pass;
156282
rows.push(
157-
`| ${provider === "codex" ? "Codex" : "Claude"} | ${metric.label} | ${baselineObserved === undefined ? "—" : formatValue(baselineObserved, metric.kind)} | ${formatValue(observed, metric.kind)} | ${formatImpact(observed, baselineObserved, metric.kind)} | ${formatValue(ceiling, metric.kind)} | ${pass ? "✅" : "❌"} |`,
283+
`| ${providerLabel(provider)} | ${metric.label} | ${baselineObserved === undefined ? "—" : formatValue(baselineObserved, metric.kind)} | ${formatValue(observed, metric.kind)} | ${formatImpact(observed, baselineObserved, metric.kind)} | ${formatValue(ceiling, metric.kind)} | ${pass ? "✅" : "❌"} |`,
158284
);
159285

160286
if (comparable && baseline.providers[provider].ceiling[metric.key] !== ceiling) {
161287
ceilingChanges.push(
162-
`- ${provider === "codex" ? "Codex" : "Claude"} ${metric.label}: ${formatValue(baseline.providers[provider].ceiling[metric.key], metric.kind)} → ${formatValue(ceiling, metric.kind)}`,
288+
`- ${providerLabel(provider)} ${metric.label}: ${formatValue(baseline.providers[provider].ceiling[metric.key], metric.kind)} → ${formatValue(ceiling, metric.kind)}`,
163289
);
164290
}
165291
}
166292
}
293+
const scenarios = renderScenarios(
294+
input.currentScenarios,
295+
input.baselineScenarios,
296+
ceilingChanges,
297+
);
298+
failed ||= scenarios.failed;
167299

168300
const baselineLink = input.baselineRun
169301
? `[\`${input.baselineRun.sha.slice(0, 7)}\`](${input.baselineRun.url})`
@@ -203,6 +335,7 @@ function renderComment(input) {
203335
"| --- | --- | ---: | ---: | ---: | ---: | --- |",
204336
...rows,
205337
"",
338+
...scenarios.lines,
206339
`Baseline: ${baselineLink} · PR result: ${currentLink} · Source CI: ${input.currentRun.conclusion}`,
207340
"",
208341
"<details>",
@@ -212,7 +345,7 @@ function renderComment(input) {
212345
"",
213346
...PROVIDERS.map(
214347
(provider) =>
215-
`- ${provider === "codex" ? "Codex" : "Claude"} decoded thread snapshot: ${formatBytes(current.providers[provider].observed.threadSnapshotDecodedBytes)}`,
348+
`- ${providerLabel(provider)} decoded thread snapshot: ${formatBytes(current.providers[provider].observed.threadSnapshotDecodedBytes)}`,
216349
),
217350
"",
218351
"</details>",
@@ -437,7 +570,14 @@ async function publish({ github, context, core }) {
437570
url: `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${process.env.BASELINE_RUN_ID}`,
438571
}
439572
: undefined;
440-
const body = renderComment({ current, baseline, currentRun, baselineRun });
573+
const body = renderComment({
574+
current,
575+
baseline,
576+
currentScenarios: readScenarios(process.env.PR_RESULT_DIR),
577+
baselineScenarios: readScenarios(process.env.BASELINE_RESULT_DIR),
578+
currentRun,
579+
baselineRun,
580+
});
441581
const published = await upsertCommentForCurrentHead(
442582
github,
443583
context,
@@ -458,4 +598,5 @@ module.exports = {
458598
resolve,
459599
upsertCommentForCurrentHead,
460600
validateResult,
601+
validateScenarios,
461602
};

‎.github/scripts/thread-transfer-report.test.cjs‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,11 @@ const assert = require("node:assert/strict");
22
const test = require("node:test");
33

44
const {
5+
renderComment,
56
resolve,
67
upsertCommentForCurrentHead,
78
validateResult,
9+
validateScenarios,
810
} = require("./thread-transfer-report.cjs");
911

1012
function result(overrides = {}) {
@@ -40,6 +42,29 @@ function result(overrides = {}) {
4042
};
4143
}
4244

45+
function scenarios(overrides = {}) {
46+
const rows = [
47+
{
48+
phase: "long turn finished, cold open",
49+
metric: "HTTP decoded",
50+
unit: "bytes",
51+
observed: 390_000,
52+
ceiling: 420_000,
53+
},
54+
{
55+
phase: "idle thread resume",
56+
metric: "WebSocket messages",
57+
unit: "count",
58+
observed: 1,
59+
ceiling: 1,
60+
},
61+
];
62+
return {
63+
schemaVersion: 1,
64+
providers: { codex: [{ ...rows[0], ...overrides }, rows[1]], claudeAgent: rows },
65+
};
66+
}
67+
4368
test("validates the fixed artifact schema", () => {
4469
assert.equal(validateResult(result()).schemaVersion, 1);
4570
assert.throws(
@@ -50,6 +75,60 @@ test("validates the fixed artifact schema", () => {
5075
() => validateResult(result({ totalWireBytes: "lots" })),
5176
/non-negative safe integer/,
5277
);
78+
79+
assert.equal(validateScenarios(scenarios()).schemaVersion, 1);
80+
for (const phase of ["@everyone", "[link](https://example.com)", "a | b", "<img>", ""]) {
81+
assert.throws(() => validateScenarios(scenarios({ phase })), /short plain label/);
82+
}
83+
assert.throws(() => validateScenarios(scenarios({ note: "extra" })), /unexpected fields/);
84+
assert.throws(() => validateScenarios(scenarios({ ceiling: null })), /non-negative safe integer/);
85+
assert.throws(
86+
() =>
87+
validateScenarios(scenarios({ phase: "idle thread resume", metric: "WebSocket messages" })),
88+
/repeats an earlier row/,
89+
);
90+
});
91+
92+
test("lists scenario rows that changed or failed above the collapsed table", () => {
93+
const run = { sha: "abcdef1234", url: "https://example.com/run", conclusion: "success" };
94+
const render = (currentScenarios, baselineScenarios) =>
95+
renderComment({
96+
current: result(),
97+
baseline: result(),
98+
currentScenarios,
99+
baselineScenarios,
100+
currentRun: run,
101+
baselineRun: { ...run, matchesBase: true },
102+
});
103+
const open = (body) => body.slice(0, body.indexOf("<details>"));
104+
105+
const unchanged = render(scenarios(), scenarios());
106+
assert.match(unchanged, /✅ Thread transfer remains within every enforced ceiling/);
107+
assert.match(open(unchanged), /All 4 scenario measurements match `main`/);
108+
assert.doesNotMatch(open(unchanged), /long turn finished, cold open/);
109+
110+
const grown = render(scenarios({ observed: 400_000 }), scenarios());
111+
assert.match(open(grown), /1 of 4 scenario measurements changed or failed/);
112+
assert.match(
113+
open(grown),
114+
/\| Codex \| `long turn finished, cold open` \| `HTTP decoded` \| 380\.9 KiB \| 390\.6 KiB \| \+9\.8 KiB \(\+2\.6%\) \| 410\.2 KiB \| ✅ \|/,
115+
);
116+
117+
const over = render(scenarios({ observed: 430_000, ceiling: 425_000 }), scenarios());
118+
assert.match(over, /❌ One or more thread transfer ceilings were exceeded/);
119+
assert.match(
120+
over,
121+
/Codex `long turn finished, cold open` `HTTP decoded`: 410\.2 KiB → 415\.0 KiB/,
122+
);
123+
124+
// A row `main` does not have is not reported as matching it.
125+
const added = render(scenarios({ phase: "GH-26" }), scenarios());
126+
assert.match(open(added), /3 scenario measurements match `main`\. 1 have no `main` baseline\./);
127+
assert.match(added, /\| Codex \| `GH-26` \|/);
128+
129+
// A main run from before the scenario file has nothing to compare; a PR run from before it has no section.
130+
assert.match(open(render(scenarios(), undefined)), /No `main` baseline exists yet/);
131+
assert.doesNotMatch(render(undefined, scenarios()), /Other scenarios/);
53132
});
54133

55134
test("resolves a fallback PR with a redacted head repo and exact main baseline", async () => {

0 commit comments

Comments
 (0)