Skip to content

Commit 57de417

Browse files
committed
ci: comment thread transfer impact on pull requests
1 parent 49c1ec1 commit 57de417

6 files changed

Lines changed: 641 additions & 1 deletion

File tree

Lines changed: 368 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,368 @@
1+
const fs = require("node:fs");
2+
const path = require("node:path");
3+
4+
const ARTIFACT_NAME = "thread-transfer-results";
5+
const RESULT_FILE = "thread-transfer-result.json";
6+
const COMMENT_MARKER = "<!-- t3-thread-transfer-report -->";
7+
const PROVIDERS = ["codex", "claudeAgent"];
8+
const OBSERVED_KEYS = [
9+
"totalWireBytes",
10+
"threadSnapshotWireBytes",
11+
"threadSnapshotDecodedBytes",
12+
"measuredTurnWebSocketWireBytes",
13+
"measuredTurnWebSocketDecodedBytes",
14+
"measuredTurnWebSocketMessages",
15+
];
16+
const CEILING_KEYS = [
17+
"totalWireBytes",
18+
"threadSnapshotWireBytes",
19+
"measuredTurnWebSocketWireBytes",
20+
"measuredTurnWebSocketDecodedBytes",
21+
"measuredTurnWebSocketMessages",
22+
];
23+
const SCENARIO_KEYS = [
24+
"id",
25+
"historyTurns",
26+
"historyCommandToolsPerTurn",
27+
"historyMcpResultBytes",
28+
"measuredCommandTools",
29+
"measuredMcpResultBytes",
30+
];
31+
32+
function assertObject(value, label) {
33+
if (value === null || typeof value !== "object" || Array.isArray(value)) {
34+
throw new Error(`${label} must be an object`);
35+
}
36+
}
37+
38+
function assertExactKeys(value, expected, label) {
39+
assertObject(value, label);
40+
const actual = Object.keys(value).sort();
41+
const wanted = [...expected].sort();
42+
if (actual.length !== wanted.length || actual.some((key, index) => key !== wanted[index])) {
43+
throw new Error(`${label} has unexpected fields`);
44+
}
45+
}
46+
47+
function assertMetric(value, label) {
48+
if (!Number.isSafeInteger(value) || value < 0 || value > 1_000_000_000) {
49+
throw new Error(`${label} must be a non-negative safe integer below 1,000,000,000`);
50+
}
51+
}
52+
53+
function validateResult(value) {
54+
assertExactKeys(value, ["schemaVersion", "scenario", "providers"], "result");
55+
if (value.schemaVersion !== 1) {
56+
throw new Error("result.schemaVersion must be 1");
57+
}
58+
59+
assertExactKeys(value.scenario, SCENARIO_KEYS, "result.scenario");
60+
if (value.scenario.id !== "thread-transfer-v1") {
61+
throw new Error("result.scenario.id is not supported");
62+
}
63+
for (const key of SCENARIO_KEYS.slice(1)) {
64+
assertMetric(value.scenario[key], `result.scenario.${key}`);
65+
}
66+
67+
assertExactKeys(value.providers, PROVIDERS, "result.providers");
68+
for (const provider of PROVIDERS) {
69+
const entry = value.providers[provider];
70+
assertExactKeys(entry, ["observed", "ceiling"], `result.providers.${provider}`);
71+
assertExactKeys(entry.observed, OBSERVED_KEYS, `result.providers.${provider}.observed`);
72+
assertExactKeys(entry.ceiling, CEILING_KEYS, `result.providers.${provider}.ceiling`);
73+
for (const key of OBSERVED_KEYS) {
74+
assertMetric(entry.observed[key], `result.providers.${provider}.observed.${key}`);
75+
}
76+
for (const key of CEILING_KEYS) {
77+
assertMetric(entry.ceiling[key], `result.providers.${provider}.ceiling.${key}`);
78+
}
79+
}
80+
81+
return value;
82+
}
83+
84+
function readResult(directory) {
85+
if (!directory) return undefined;
86+
const file = path.join(directory, RESULT_FILE);
87+
if (!fs.existsSync(file)) return undefined;
88+
const stat = fs.lstatSync(file);
89+
if (!stat.isFile() || stat.size > 64 * 1_024) {
90+
throw new Error("thread transfer result must be a regular file smaller than 64 KiB");
91+
}
92+
return validateResult(JSON.parse(fs.readFileSync(file, "utf8")));
93+
}
94+
95+
function formatBytes(bytes) {
96+
if (bytes < 1_024) return `${bytes} B`;
97+
if (bytes >= 1_024 * 1_024) return `${(bytes / 1_024 / 1_024).toFixed(2)} MiB`;
98+
return `${(bytes / 1_024).toFixed(1)} KiB`;
99+
}
100+
101+
function formatValue(value, kind) {
102+
return kind === "messages" ? value.toLocaleString("en-US") : formatBytes(value);
103+
}
104+
105+
function formatImpact(current, baseline, kind) {
106+
if (baseline === undefined) return "—";
107+
const delta = current - baseline;
108+
const prefix = delta > 0 ? "+" : delta < 0 ? "−" : "";
109+
const magnitude = formatValue(Math.abs(delta), kind);
110+
const percent =
111+
baseline === 0 ? "" : ` (${prefix}${Math.abs((delta / baseline) * 100).toFixed(1)}%)`;
112+
return `${prefix}${magnitude}${percent}`;
113+
}
114+
115+
function sameScenario(left, right) {
116+
return SCENARIO_KEYS.every((key) => left[key] === right[key]);
117+
}
118+
119+
const METRICS = [
120+
{ key: "totalWireBytes", label: "Total thread wire", kind: "bytes" },
121+
{ key: "threadSnapshotWireBytes", label: "Thread snapshot wire", kind: "bytes" },
122+
{
123+
key: "measuredTurnWebSocketWireBytes",
124+
label: "Live turn WebSocket wire",
125+
kind: "bytes",
126+
},
127+
{
128+
key: "measuredTurnWebSocketDecodedBytes",
129+
label: "Live turn WebSocket decoded",
130+
kind: "bytes",
131+
},
132+
{ key: "measuredTurnWebSocketMessages", label: "Live turn messages", kind: "messages" },
133+
];
134+
135+
function renderComment(input) {
136+
const current = input.current;
137+
const baseline = input.baseline;
138+
const comparable = baseline !== undefined && sameScenario(current.scenario, baseline.scenario);
139+
const rows = [];
140+
const ceilingChanges = [];
141+
let failed = false;
142+
143+
for (const provider of PROVIDERS) {
144+
for (const metric of METRICS) {
145+
const observed = current.providers[provider].observed[metric.key];
146+
const ceiling = current.providers[provider].ceiling[metric.key];
147+
const baselineObserved = comparable
148+
? baseline.providers[provider].observed[metric.key]
149+
: undefined;
150+
const pass = observed <= ceiling;
151+
failed ||= !pass;
152+
rows.push(
153+
`| ${provider === "codex" ? "Codex" : "Claude"} | ${metric.label} | ${baselineObserved === undefined ? "—" : formatValue(baselineObserved, metric.kind)} | ${formatValue(observed, metric.kind)} | ${formatImpact(observed, baselineObserved, metric.kind)} | ${formatValue(ceiling, metric.kind)} | ${pass ? "✅" : "❌"} |`,
154+
);
155+
156+
if (baseline && baseline.providers[provider].ceiling[metric.key] !== ceiling) {
157+
ceilingChanges.push(
158+
`- ${provider === "codex" ? "Codex" : "Claude"} ${metric.label}: ${formatValue(baseline.providers[provider].ceiling[metric.key], metric.kind)} → ${formatValue(ceiling, metric.kind)}`,
159+
);
160+
}
161+
}
162+
}
163+
164+
const baselineLink = input.baselineRun
165+
? `[\`${input.baselineRun.sha.slice(0, 7)}\`](${input.baselineRun.url})`
166+
: "unavailable";
167+
const currentLink = `[\`${input.currentRun.sha.slice(0, 7)}\`](${input.currentRun.url})`;
168+
const notices = [];
169+
if (!baseline) {
170+
notices.push(
171+
"> ℹ️ No successful `main` baseline artifact is available yet. This run establishes the initial measurement.",
172+
);
173+
} else if (!comparable) {
174+
notices.push(
175+
"> ⚠️ The thread fixture changed, so impact percentages are not directly comparable to the `main` baseline.",
176+
);
177+
} else if (!input.baselineRun.matchesBase) {
178+
notices.push(
179+
"> ℹ️ The exact PR base did not have a successful artifact. Baseline uses the latest successful `main` measurement shown below.",
180+
);
181+
}
182+
if (ceilingChanges.length > 0) {
183+
notices.push(
184+
`> ⚠️ **This PR changes transfer ceilings:**\n>\n${ceilingChanges.map((line) => `> ${line}`).join("\n")}`,
185+
);
186+
}
187+
188+
return [
189+
COMMENT_MARKER,
190+
"## Thread transfer impact",
191+
"",
192+
failed
193+
? "❌ One or more thread transfer ceilings were exceeded."
194+
: "✅ Thread transfer remains within every enforced ceiling.",
195+
...(notices.length > 0 ? ["", ...notices] : []),
196+
"",
197+
"| Provider | Metric | Main baseline | This PR | Impact | PR ceiling | |",
198+
"| --- | --- | ---: | ---: | ---: | ---: | --- |",
199+
...rows,
200+
"",
201+
`Baseline: ${baselineLink} · PR result: ${currentLink} · Source CI: ${input.currentRun.conclusion}`,
202+
"",
203+
"<details>",
204+
"<summary>Scenario and decoded snapshot size</summary>",
205+
"",
206+
`${current.scenario.historyTurns} historical turns, ${current.scenario.historyCommandToolsPerTurn} command tools per turn, ${formatBytes(current.scenario.historyMcpResultBytes)} retained MCP result per historical turn, and a ${formatBytes(current.scenario.measuredMcpResultBytes)} retained result in the measured turn.`,
207+
"",
208+
...PROVIDERS.map(
209+
(provider) =>
210+
`- ${provider === "codex" ? "Codex" : "Claude"} decoded thread snapshot: ${formatBytes(current.providers[provider].observed.threadSnapshotDecodedBytes)}`,
211+
),
212+
"",
213+
"</details>",
214+
"",
215+
"_Updated in place by a trusted workflow. PR artifacts are strictly validated and never executed._",
216+
].join("\n");
217+
}
218+
219+
async function artifactsForRun(github, owner, repo, runId) {
220+
return github.paginate(github.rest.actions.listWorkflowRunArtifacts, {
221+
owner,
222+
repo,
223+
run_id: runId,
224+
per_page: 100,
225+
});
226+
}
227+
228+
function findResultArtifact(artifacts) {
229+
return artifacts.find((artifact) => artifact.name === ARTIFACT_NAME && !artifact.expired);
230+
}
231+
232+
async function resolve({ github, context, core }) {
233+
const source = context.payload.workflow_run;
234+
const { owner, repo } = context.repo;
235+
if (source.event !== "pull_request") {
236+
core.setOutput("publish", "false");
237+
return;
238+
}
239+
240+
let pullNumber = source.pull_requests?.[0]?.number;
241+
if (!pullNumber) {
242+
const associated = await github.paginate(
243+
github.rest.repos.listPullRequestsAssociatedWithCommit,
244+
{ owner, repo, commit_sha: source.head_sha, per_page: 100 },
245+
);
246+
pullNumber = associated.find((pull) => pull.state === "open")?.number;
247+
}
248+
if (!pullNumber) {
249+
core.info("No open pull request is associated with the completed CI run.");
250+
core.setOutput("publish", "false");
251+
return;
252+
}
253+
254+
const { data: pull } = await github.rest.pulls.get({ owner, repo, pull_number: pullNumber });
255+
if (pull.head.sha !== source.head_sha) {
256+
core.info(`Skipping stale CI result ${source.head_sha}; PR head is ${pull.head.sha}.`);
257+
core.setOutput("publish", "false");
258+
return;
259+
}
260+
261+
const sourceArtifacts = await artifactsForRun(github, owner, repo, source.id);
262+
const sourceArtifact = findResultArtifact(sourceArtifacts);
263+
const workflowRuns = await github.paginate(github.rest.actions.listWorkflowRuns, {
264+
owner,
265+
repo,
266+
workflow_id: source.workflow_id,
267+
branch: pull.base.ref,
268+
event: "push",
269+
status: "success",
270+
per_page: 100,
271+
});
272+
const orderedRuns = [
273+
...workflowRuns.filter((run) => run.head_sha === pull.base.sha),
274+
...workflowRuns.filter((run) => run.head_sha !== pull.base.sha),
275+
].slice(0, 20);
276+
277+
let baselineRun;
278+
for (const run of orderedRuns) {
279+
const artifacts = await artifactsForRun(github, owner, repo, run.id);
280+
if (findResultArtifact(artifacts)) {
281+
baselineRun = run;
282+
break;
283+
}
284+
}
285+
286+
core.setOutput("publish", "true");
287+
core.setOutput("pull_number", String(pullNumber));
288+
core.setOutput("pr_artifact", sourceArtifact ? "true" : "false");
289+
core.setOutput("pr_run_id", String(source.id));
290+
core.setOutput("pr_sha", source.head_sha);
291+
core.setOutput("pr_conclusion", source.conclusion ?? "unknown");
292+
core.setOutput("baseline_artifact", baselineRun ? "true" : "false");
293+
core.setOutput("baseline_run_id", baselineRun ? String(baselineRun.id) : "");
294+
core.setOutput("baseline_sha", baselineRun?.head_sha ?? "");
295+
core.setOutput(
296+
"baseline_matches_base",
297+
baselineRun?.head_sha === pull.base.sha ? "true" : "false",
298+
);
299+
}
300+
301+
async function upsertComment(github, context, pullNumber, body) {
302+
const { owner, repo } = context.repo;
303+
const comments = await github.paginate(github.rest.issues.listComments, {
304+
owner,
305+
repo,
306+
issue_number: pullNumber,
307+
per_page: 100,
308+
});
309+
const existing = comments.find(
310+
(comment) =>
311+
comment.user?.login === "github-actions[bot]" && comment.body?.includes(COMMENT_MARKER),
312+
);
313+
if (existing) {
314+
await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body });
315+
} else {
316+
await github.rest.issues.createComment({ owner, repo, issue_number: pullNumber, body });
317+
}
318+
}
319+
320+
async function publish({ github, context, core }) {
321+
const pullNumber = Number(process.env.PR_NUMBER);
322+
if (!Number.isSafeInteger(pullNumber) || pullNumber <= 0) {
323+
throw new Error("PR_NUMBER is invalid");
324+
}
325+
326+
const current = readResult(process.env.PR_RESULT_DIR);
327+
const currentRun = {
328+
sha: process.env.PR_SHA,
329+
conclusion: process.env.PR_CONCLUSION,
330+
url: `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${process.env.PR_RUN_ID}`,
331+
};
332+
if (!current) {
333+
await upsertComment(
334+
github,
335+
context,
336+
pullNumber,
337+
[
338+
COMMENT_MARKER,
339+
"## Thread transfer impact",
340+
"",
341+
`⚠️ The latest [CI run](${currentRun.url}) did not produce a thread transfer result for \`${currentRun.sha.slice(0, 7)}\`.`,
342+
"",
343+
"_This comment will update automatically after the next completed run._",
344+
].join("\n"),
345+
);
346+
return;
347+
}
348+
349+
const baseline = readResult(process.env.BASELINE_RESULT_DIR);
350+
const baselineRun = baseline
351+
? {
352+
sha: process.env.BASELINE_SHA,
353+
matchesBase: process.env.BASELINE_MATCHES_BASE === "true",
354+
url: `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${process.env.BASELINE_RUN_ID}`,
355+
}
356+
: undefined;
357+
const body = renderComment({ current, baseline, currentRun, baselineRun });
358+
await upsertComment(github, context, pullNumber, body);
359+
core.info(`Updated thread transfer report on PR #${pullNumber}.`);
360+
}
361+
362+
module.exports = {
363+
publish,
364+
readResult,
365+
renderComment,
366+
resolve,
367+
validateResult,
368+
};

0 commit comments

Comments
 (0)