@@ -3,6 +3,7 @@ const path = require("node:path");
33
44const ARTIFACT_NAME = "thread-transfer-results" ;
55const RESULT_FILE = "thread-transfer-result.json" ;
6+ const SCENARIOS_FILE = "thread-transfer-scenarios.json" ;
67const COMMENT_MARKER = "<!-- t3-thread-transfer-report -->" ;
78const PROVIDERS = [ "codex" , "claudeAgent" ] ;
89const OBSERVED_KEYS = [
@@ -29,6 +30,12 @@ const SCENARIO_KEYS = [
2930 "measuredMcpResultBytes" ,
3031] ;
3132
33+ const SCENARIO_ROW_KEYS = [ "phase" , "metric" , "unit" , "observed" , "ceiling" ] ;
34+ // These labels come from the PR and reach a comment the bot posts: no markup, links or mentions.
35+ // GitHub still links plain text such as `GH-26`, so they are rendered as code spans.
36+ const SCENARIO_LABEL = / ^ [ A - Z a - z 0 - 9 ] [ A - Z a - z 0 - 9 , - ] { 0 , 79 } $ / ;
37+ const MAX_SCENARIO_ROWS = 100 ;
38+
3239function resultShaMarker ( sha ) {
3340 return `<!-- t3-thread-transfer-result-sha:${ sha } -->` ;
3441}
@@ -85,15 +92,63 @@ function validateResult(value) {
8592 return value ;
8693}
8794
88- function readResult ( directory ) {
95+ function validateScenarios ( value ) {
96+ assertExactKeys ( value , [ "schemaVersion" , "providers" ] , "scenarios" ) ;
97+ if ( value . schemaVersion !== 1 ) {
98+ throw new Error ( "scenarios.schemaVersion must be 1" ) ;
99+ }
100+ assertExactKeys ( value . providers , PROVIDERS , "scenarios.providers" ) ;
101+ for ( const provider of PROVIDERS ) {
102+ const rows = value . providers [ provider ] ;
103+ if ( ! Array . isArray ( rows ) || rows . length > MAX_SCENARIO_ROWS ) {
104+ throw new Error (
105+ `scenarios.providers.${ provider } must list at most ${ MAX_SCENARIO_ROWS } rows` ,
106+ ) ;
107+ }
108+ const seen = new Set ( ) ;
109+ for ( const [ index , row ] of rows . entries ( ) ) {
110+ const label = `scenarios.providers.${ provider } [${ index } ]` ;
111+ assertExactKeys ( row , SCENARIO_ROW_KEYS , label ) ;
112+ for ( const key of [ "phase" , "metric" ] ) {
113+ if ( typeof row [ key ] !== "string" || ! SCENARIO_LABEL . test ( row [ key ] ) ) {
114+ throw new Error ( `${ label } .${ key } must be a short plain label` ) ;
115+ }
116+ }
117+ if ( row . unit !== "bytes" && row . unit !== "count" ) {
118+ throw new Error ( `${ label } .unit must be bytes or count` ) ;
119+ }
120+ assertMetric ( row . observed , `${ label } .observed` ) ;
121+ assertMetric ( row . ceiling , `${ label } .ceiling` ) ;
122+ const key = scenarioKey ( row ) ;
123+ if ( seen . has ( key ) ) throw new Error ( `${ label } repeats an earlier row` ) ;
124+ seen . add ( key ) ;
125+ }
126+ }
127+ return value ;
128+ }
129+
130+ function scenarioKey ( row ) {
131+ return `${ row . phase } |${ row . metric } ` ;
132+ }
133+
134+ function readArtifactFile ( directory , name , validate ) {
89135 if ( ! directory ) return undefined ;
90- const file = path . join ( directory , RESULT_FILE ) ;
136+ const file = path . join ( directory , name ) ;
91137 if ( ! fs . existsSync ( file ) ) return undefined ;
92138 const stat = fs . lstatSync ( file ) ;
93139 if ( ! stat . isFile ( ) || stat . size > 64 * 1_024 ) {
94- throw new Error ( "thread transfer result must be a regular file smaller than 64 KiB" ) ;
140+ throw new Error ( ` ${ name } must be a regular file smaller than 64 KiB` ) ;
95141 }
96- return validateResult ( JSON . parse ( fs . readFileSync ( file , "utf8" ) ) ) ;
142+ return validate ( JSON . parse ( fs . readFileSync ( file , "utf8" ) ) ) ;
143+ }
144+
145+ function readResult ( directory ) {
146+ return readArtifactFile ( directory , RESULT_FILE , validateResult ) ;
147+ }
148+
149+ // Older runs upload only the result file, so a run without scenarios is still a valid run.
150+ function readScenarios ( directory ) {
151+ return readArtifactFile ( directory , SCENARIOS_FILE , validateScenarios ) ;
97152}
98153
99154function formatBytes ( bytes ) {
@@ -103,7 +158,7 @@ function formatBytes(bytes) {
103158}
104159
105160function formatValue ( value , kind ) {
106- return kind === "messages " ? value . toLocaleString ( "en-US" ) : formatBytes ( value ) ;
161+ return kind === "bytes " ? formatBytes ( value ) : value . toLocaleString ( "en-US" ) ;
107162}
108163
109164function formatImpact ( current , baseline , kind ) {
@@ -136,6 +191,77 @@ const METRICS = [
136191 { key : "measuredTurnWebSocketMessages" , label : "Live turn messages" , kind : "messages" } ,
137192] ;
138193
194+ function providerLabel ( provider ) {
195+ return provider === "codex" ? "Codex" : "Claude" ;
196+ }
197+
198+ /**
199+ * The scenario rows of a run against the same rows on `main`. Rows that fail or differ from
200+ * `main` are listed in the open; the full table is collapsed below them.
201+ */
202+ function renderScenarios ( current , baseline , ceilingChanges ) {
203+ if ( ! current ) return { lines : [ ] , failed : false } ;
204+ const header = [
205+ "| Provider | Scenario | Metric | Main baseline | This PR | Impact | PR ceiling | |" ,
206+ "| --- | --- | --- | ---: | ---: | ---: | ---: | --- |" ,
207+ ] ;
208+ const all = [ ] ;
209+ const notable = [ ] ;
210+ let failed = false ;
211+ let withoutBaseline = 0 ;
212+
213+ for ( const provider of PROVIDERS ) {
214+ const baselineRows = new Map (
215+ ( baseline ?. providers [ provider ] ?? [ ] ) . map ( ( row ) => [ scenarioKey ( row ) , row ] ) ,
216+ ) ;
217+ for ( const row of current . providers [ provider ] ) {
218+ const before = baselineRows . get ( scenarioKey ( row ) ) ;
219+ const beforeObserved = before ?. unit === row . unit ? before . observed : undefined ;
220+ const pass = row . observed <= row . ceiling ;
221+ failed ||= ! pass ;
222+ if ( beforeObserved === undefined ) withoutBaseline += 1 ;
223+ const line = `| ${ providerLabel ( provider ) } | \`${ row . phase } \` | \`${ row . metric } \` | ${ beforeObserved === undefined ? "—" : formatValue ( beforeObserved , row . unit ) } | ${ formatValue ( row . observed , row . unit ) } | ${ formatImpact ( row . observed , beforeObserved , row . unit ) } | ${ formatValue ( row . ceiling , row . unit ) } | ${ pass ? "✅" : "❌" } |` ;
224+ all . push ( line ) ;
225+ if ( ! pass || ( beforeObserved !== undefined && beforeObserved !== row . observed ) ) {
226+ notable . push ( line ) ;
227+ }
228+ if ( beforeObserved !== undefined && before . ceiling !== row . ceiling ) {
229+ ceilingChanges . push (
230+ `- ${ providerLabel ( provider ) } \`${ row . phase } \` \`${ row . metric } \`: ${ formatValue ( before . ceiling , row . unit ) } → ${ formatValue ( row . ceiling , row . unit ) } ` ,
231+ ) ;
232+ }
233+ }
234+ }
235+
236+ const compared = all . length - withoutBaseline ;
237+ const summary = ! baseline
238+ ? `No \`main\` baseline exists yet for these ${ all . length } scenario measurements.`
239+ : [
240+ notable . length > 0
241+ ? `${ notable . length } of ${ all . length } scenario measurements changed or failed.`
242+ : `${ compared === all . length ? "All " : "" } ${ compared } scenario measurements match \`main\`.` ,
243+ ...( withoutBaseline > 0 ? [ `${ withoutBaseline } have no \`main\` baseline.` ] : [ ] ) ,
244+ ] . join ( " " ) ;
245+ return {
246+ failed,
247+ lines : [
248+ "### Other scenarios" ,
249+ "" ,
250+ summary ,
251+ ...( notable . length > 0 ? [ "" , ...header , ...notable ] : [ ] ) ,
252+ "" ,
253+ "<details>" ,
254+ `<summary>All ${ all . length } scenario measurements</summary>` ,
255+ "" ,
256+ ...header ,
257+ ...all ,
258+ "" ,
259+ "</details>" ,
260+ "" ,
261+ ] ,
262+ } ;
263+ }
264+
139265function renderComment ( input ) {
140266 const current = input . current ;
141267 const baseline = input . baseline ;
@@ -154,16 +280,22 @@ function renderComment(input) {
154280 const pass = observed <= ceiling ;
155281 failed ||= ! pass ;
156282 rows . push (
157- `| ${ provider === "codex" ? "Codex" : "Claude" } | ${ metric . label } | ${ baselineObserved === undefined ? "—" : formatValue ( baselineObserved , metric . kind ) } | ${ formatValue ( observed , metric . kind ) } | ${ formatImpact ( observed , baselineObserved , metric . kind ) } | ${ formatValue ( ceiling , metric . kind ) } | ${ pass ? "✅" : "❌" } |` ,
283+ `| ${ providerLabel ( provider ) } | ${ metric . label } | ${ baselineObserved === undefined ? "—" : formatValue ( baselineObserved , metric . kind ) } | ${ formatValue ( observed , metric . kind ) } | ${ formatImpact ( observed , baselineObserved , metric . kind ) } | ${ formatValue ( ceiling , metric . kind ) } | ${ pass ? "✅" : "❌" } |` ,
158284 ) ;
159285
160286 if ( comparable && baseline . providers [ provider ] . ceiling [ metric . key ] !== ceiling ) {
161287 ceilingChanges . push (
162- `- ${ provider === "codex" ? "Codex" : "Claude" } ${ metric . label } : ${ formatValue ( baseline . providers [ provider ] . ceiling [ metric . key ] , metric . kind ) } → ${ formatValue ( ceiling , metric . kind ) } ` ,
288+ `- ${ providerLabel ( provider ) } ${ metric . label } : ${ formatValue ( baseline . providers [ provider ] . ceiling [ metric . key ] , metric . kind ) } → ${ formatValue ( ceiling , metric . kind ) } ` ,
163289 ) ;
164290 }
165291 }
166292 }
293+ const scenarios = renderScenarios (
294+ input . currentScenarios ,
295+ input . baselineScenarios ,
296+ ceilingChanges ,
297+ ) ;
298+ failed ||= scenarios . failed ;
167299
168300 const baselineLink = input . baselineRun
169301 ? `[\`${ input . baselineRun . sha . slice ( 0 , 7 ) } \`](${ input . baselineRun . url } )`
@@ -203,6 +335,7 @@ function renderComment(input) {
203335 "| --- | --- | ---: | ---: | ---: | ---: | --- |" ,
204336 ...rows ,
205337 "" ,
338+ ...scenarios . lines ,
206339 `Baseline: ${ baselineLink } · PR result: ${ currentLink } · Source CI: ${ input . currentRun . conclusion } ` ,
207340 "" ,
208341 "<details>" ,
@@ -212,7 +345,7 @@ function renderComment(input) {
212345 "" ,
213346 ...PROVIDERS . map (
214347 ( provider ) =>
215- `- ${ provider === "codex" ? "Codex" : "Claude" } decoded thread snapshot: ${ formatBytes ( current . providers [ provider ] . observed . threadSnapshotDecodedBytes ) } ` ,
348+ `- ${ providerLabel ( provider ) } decoded thread snapshot: ${ formatBytes ( current . providers [ provider ] . observed . threadSnapshotDecodedBytes ) } ` ,
216349 ) ,
217350 "" ,
218351 "</details>" ,
@@ -437,7 +570,14 @@ async function publish({ github, context, core }) {
437570 url : `https://github.com/${ context . repo . owner } /${ context . repo . repo } /actions/runs/${ process . env . BASELINE_RUN_ID } ` ,
438571 }
439572 : undefined ;
440- const body = renderComment ( { current, baseline, currentRun, baselineRun } ) ;
573+ const body = renderComment ( {
574+ current,
575+ baseline,
576+ currentScenarios : readScenarios ( process . env . PR_RESULT_DIR ) ,
577+ baselineScenarios : readScenarios ( process . env . BASELINE_RESULT_DIR ) ,
578+ currentRun,
579+ baselineRun,
580+ } ) ;
441581 const published = await upsertCommentForCurrentHead (
442582 github ,
443583 context ,
@@ -458,4 +598,5 @@ module.exports = {
458598 resolve,
459599 upsertCommentForCurrentHead,
460600 validateResult,
601+ validateScenarios,
461602} ;
0 commit comments