Skip to content

Commit ac223be

Browse files
authored
fix(benchmarks): limit targeted bundle reports to measured scenarios (#8456)
1 parent 2fb7723 commit ac223be

4 files changed

Lines changed: 155 additions & 13 deletions

File tree

‎benchmarks/bundle-size/README.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,8 @@ Without `--name`, runs replace `current.json` and the logs for the steps they ru
114114
Emitted bundles still use the shared `dist/` directory. To retain emitted bundles for a candidate, pass a separate `--dist-dir`.
115115

116116
`--baseline` accepts a run name or a JSON file path, such as `--baseline ./baseline.json`.
117+
With `--scenario`, the comparison reports only the scenarios measured in the current run, even when the baseline contains more scenarios. Selected scenarios missing from the baseline remain visible with `n/a` for their baseline and deltas.
118+
Without `--scenario`, comparisons include scenarios from either result file.
117119
`--results-dir <dir>` changes the results root, including the location of named runs.
118120
Use the same scenario selection and measurement flags for the baseline and candidate.
119121

@@ -131,6 +133,8 @@ pnpm benchmark:bundle-size:diff --baseline /tmp/base-current.json --id react-rou
131133
pnpm benchmark:bundle-size:history --id react-router.minimal --top-deltas 20
132134
```
133135

136+
Use `pnpm benchmark:bundle-size:diff --baseline /tmp/base-current.json --current-only` to compare only scenarios present in the current results. This applies to both text and `--json` output; an explicit `--id` takes precedence.
137+
134138
For source attribution, run an analysis build. This uses hidden source maps and writes source estimates into `current.json`; those estimates are for investigation only, not tracking.
135139

136140
```bash

‎scripts/benchmarks/bundle-size/diff.mjs‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ const { values } = parseArgs({
1313
default: 'benchmarks/bundle-size/results/current.json',
1414
},
1515
id: { type: 'string' },
16+
'current-only': { type: 'boolean' },
1617
json: { type: 'boolean' },
1718
},
1819
})
@@ -33,7 +34,9 @@ const baselineById = byId(readCurrent(values.baseline))
3334
const currentById = byId(readCurrent(values.current))
3435
const ids = values.id
3536
? [values.id]
36-
: [...new Set([...baselineById.keys(), ...currentById.keys()])].sort()
37+
: values['current-only']
38+
? [...currentById.keys()].sort()
39+
: [...new Set([...baselineById.keys(), ...currentById.keys()])].sort()
3740

3841
const rows = ids.map((id) => {
3942
const baseline = baselineById.get(id)

‎scripts/benchmarks/bundle-size/run.mjs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@ export function run(argv, execute = spawnSync) {
7272
Named runs use <results-root>/runs/<name>/current.json.
7373
Without --name, current.json is replaced in the results root.
7474
Logs stay beside current.json; failures print a bounded log tail.
75+
With --scenario, comparisons report only the scenarios measured in this run.
7576
Arguments after -- go to the selected unit tests, not the measurement.`)
7677
return 0
7778
}
@@ -244,6 +245,7 @@ Arguments after -- go to the selected unit tests, not the measurement.`)
244245
'--current',
245246
currentPath,
246247
...(baselinePath ? ['--baseline', baselinePath] : []),
248+
...(baselinePath && values.scenario ? ['--current-only'] : []),
247249
],
248250
false,
249251
)

‎scripts/benchmarks/bundle-size/run.test.mjs‎

Lines changed: 145 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -11,26 +11,28 @@ const scriptPath = fileURLToPath(new URL('./run.mjs', import.meta.url))
1111
const measurePath = fileURLToPath(new URL('./measure.mjs', import.meta.url))
1212

1313
/** @param {string} filePath */
14-
function writeCurrent(filePath, gzipBytes = 100) {
14+
function writeCurrent(
15+
filePath,
16+
gzipBytes = 100,
17+
ids = ['react-router.minimal'],
18+
) {
1519
fs.mkdirSync(path.dirname(filePath), { recursive: true })
1620
fs.writeFileSync(
1721
filePath,
1822
JSON.stringify({
19-
metrics: [
20-
{
21-
id: 'react-router.minimal',
22-
gzipBytes,
23-
initialGzipBytes: gzipBytes,
24-
rawBytes: gzipBytes * 3,
25-
brotliBytes: gzipBytes - 10,
26-
},
27-
],
23+
metrics: ids.map((id) => ({
24+
id,
25+
gzipBytes,
26+
initialGzipBytes: gzipBytes,
27+
rawBytes: gzipBytes * 3,
28+
brotliBytes: gzipBytes - 10,
29+
})),
2830
}),
2931
)
3032
}
3133

3234
/** @param {import('node:test').TestContext} t */
33-
function fixture(t) {
35+
function fixture(t, measuredIds = ['react-router.minimal']) {
3436
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'bundle-runner-'))
3537
t.after(() => fs.rmSync(root, { recursive: true, force: true }))
3638
/** @type {Array<{command: string, args: string[], options: import('./run.mjs').CommandOptions}>} */
@@ -52,7 +54,7 @@ function fixture(t) {
5254
if (args[0] === measurePath) {
5355
const outputDir = args[args.indexOf('--results-dir') + 1]
5456
assert.ok(outputDir)
55-
writeCurrent(path.join(outputDir, 'current.json'), 90)
57+
writeCurrent(path.join(outputDir, 'current.json'), 90, measuredIds)
5658
}
5759
if (options.stdio !== 'inherit') {
5860
fs.writeSync(options.stdio[1], 'noisy stdout\n')
@@ -187,6 +189,137 @@ test('accepts an external baseline and forwards measurement options', (t) => {
187189
assert.match(reports.join(''), /100 -> 90 \(-10\)/)
188190
})
189191

192+
test('targeted comparisons omit unrequested scenarios from a broader baseline', async (t) => {
193+
for (const [
194+
scenario,
195+
measuredIds,
196+
] of /** @type {Array<[string, string[]]>} */ ([
197+
['react-router.minimal', ['react-router.minimal']],
198+
['react-router-minimal', ['react-router.minimal']],
199+
[
200+
' solid-router.minimal, react-router.minimal ',
201+
['solid-router.minimal', 'react-router.minimal'],
202+
],
203+
])) {
204+
await t.test(scenario, (t) => {
205+
const { root, reports, execute } = fixture(t, measuredIds)
206+
writeCurrent(path.join(root, 'runs/baseline/current.json'), 100, [
207+
'react-router.full',
208+
'react-router.minimal',
209+
'solid-router.minimal',
210+
])
211+
212+
assert.equal(
213+
run(
214+
[
215+
'--results-dir',
216+
root,
217+
'--baseline',
218+
'baseline',
219+
'--scenario',
220+
scenario,
221+
],
222+
execute,
223+
),
224+
0,
225+
)
226+
assert.equal(
227+
reports.join(''),
228+
[...measuredIds]
229+
.sort()
230+
.map((id) => `${id} 100 -> 90 (-10) initial=-10 raw=-30 brotli=-10\n`)
231+
.join(''),
232+
)
233+
})
234+
}
235+
})
236+
237+
test('targeted comparisons keep measured scenarios missing from the baseline', (t) => {
238+
const { root, reports, execute } = fixture(t)
239+
writeCurrent(path.join(root, 'runs/baseline/current.json'), 100, [
240+
'react-router.full',
241+
])
242+
243+
assert.equal(
244+
run(
245+
[
246+
'--results-dir',
247+
root,
248+
'--baseline',
249+
'baseline',
250+
'--scenario',
251+
'react-router.minimal',
252+
],
253+
execute,
254+
),
255+
0,
256+
)
257+
assert.equal(
258+
reports.join(''),
259+
'react-router.minimal n/a -> 90 (n/a) initial=n/a raw=n/a brotli=n/a\n',
260+
)
261+
})
262+
263+
test('unfiltered comparisons retain baseline-only scenarios', (t) => {
264+
const { root, reports, execute } = fixture(t)
265+
writeCurrent(path.join(root, 'runs/baseline/current.json'), 100, [
266+
'react-router.full',
267+
])
268+
269+
assert.equal(
270+
run(['--results-dir', root, '--baseline', 'baseline'], execute),
271+
0,
272+
)
273+
assert.equal(
274+
reports.join(''),
275+
'react-router.full 100 -> n/a (n/a) initial=n/a raw=n/a brotli=n/a\n' +
276+
'react-router.minimal n/a -> 90 (n/a) initial=n/a raw=n/a brotli=n/a\n',
277+
)
278+
})
279+
280+
test('current-only diffs filter JSON output and preserve explicit ID selection', (t) => {
281+
const { root } = fixture(t)
282+
const baseline = path.join(root, 'baseline.json')
283+
const current = path.join(root, 'current.json')
284+
writeCurrent(baseline, 100, ['react-router.full', 'react-router.minimal'])
285+
writeCurrent(current, 90, ['solid-router.minimal', 'react-router.minimal'])
286+
const args = [
287+
fileURLToPath(new URL('./diff.mjs', import.meta.url)),
288+
'--baseline',
289+
baseline,
290+
'--current',
291+
current,
292+
'--current-only',
293+
'--json',
294+
]
295+
const result = spawnSync(process.execPath, args, { encoding: 'utf8' })
296+
assert.equal(result.status, 0, result.stderr)
297+
assert.deepEqual(JSON.parse(result.stdout), [
298+
{
299+
id: 'react-router.minimal',
300+
baseline: 100,
301+
current: 90,
302+
delta: -10,
303+
initialDelta: -10,
304+
rawDelta: -30,
305+
brotliDelta: -10,
306+
},
307+
{ id: 'solid-router.minimal', current: 90 },
308+
])
309+
310+
const selected = spawnSync(
311+
process.execPath,
312+
[...args, '--id', 'react-router.full'],
313+
{
314+
encoding: 'utf8',
315+
},
316+
)
317+
assert.equal(selected.status, 0, selected.stderr)
318+
assert.deepEqual(JSON.parse(selected.stdout), [
319+
{ id: 'react-router.full', baseline: 100 },
320+
])
321+
})
322+
190323
test('stops on test failure, preserves its exit code, and prints only a bounded log tail', (t) => {
191324
const { root, messages } = fixture(t)
192325
let callCount = 0

0 commit comments

Comments
 (0)