Skip to content

Commit dcd704e

Browse files
test(benchmark): make --list work and assert the scheduled run covers everything
--list was parsed and then ignored, so the documented way to inspect a selection ran the benchmarks instead. Implement it in the runner, and use it to assert that the shards of the full matrix — what the cron fires — list every benchmark exactly once, in both dirs.
1 parent 9870b3f commit dcd704e

3 files changed

Lines changed: 82 additions & 4 deletions

File tree

‎test/Benchmarks.unittest.js‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
"use strict";
22

3+
const { execFileSync } = require("child_process");
34
const fs = require("fs");
45
const path = require("path");
56
const { pathToFileURL } = require("url");
@@ -9,6 +10,44 @@ const { pathToFileURL } = require("url");
910

1011
const benchmarkRoot = path.join(__dirname, "benchmark");
1112
const runnerPath = path.join(benchmarkRoot, "lib", "runner.mjs");
13+
const cliPath = path.join(benchmarkRoot, "lib", "cli.mjs");
14+
const selectPath = path.join(
15+
__dirname,
16+
"..",
17+
".github",
18+
"scripts",
19+
"select-benchmarks.mjs"
20+
);
21+
22+
/**
23+
* @typedef {object} BenchmarkSelection
24+
* @property {boolean} run whether the scheduled/labeled run happens at all
25+
* @property {string} filter benchmark id filter, empty for everything
26+
* @property {{ include: { dir: string, shard: string }[] }} matrix job matrix
27+
* @property {string[]} suites selected suite ids
28+
*/
29+
30+
/**
31+
* @param {string} script absolute path of the script to run
32+
* @param {string[]} args arguments for it
33+
* @returns {string} its stdout
34+
*/
35+
const run = (script, args) =>
36+
execFileSync(process.execPath, [script, ...args], {
37+
encoding: "utf8",
38+
// the selection script appends to it, and CI sets it for every step
39+
env: { ...process.env, GITHUB_OUTPUT: "" }
40+
});
41+
42+
/**
43+
* @param {string[]} args arguments for `cli.mjs`
44+
* @returns {string[]} the benchmark ids the run would cover, sorted
45+
*/
46+
const listBenchmarks = (args) =>
47+
run(cliPath, [...args, "--list"])
48+
.split("\n")
49+
.filter(Boolean)
50+
.sort();
1251

1352
// The harness is ESM on post-20 APIs; the Deno and Bun jobs run this suite too.
1453
const itNode = process.versions.bun || process.versions.deno ? it.skip : it;
@@ -57,6 +96,33 @@ describe("Benchmarks", () => {
5796
expect(e2eFiles).not.toHaveLength(0);
5897
});
5998

99+
// the scheduled run is the only one that measures everything, and its shard
100+
// indices are computed apart from the ones cli.mjs applies
101+
itNode(
102+
"should shard the scheduled run over every benchmark",
103+
() => {
104+
const selection = /** @type {BenchmarkSelection} */ (
105+
JSON.parse(run(selectPath, ["--all"]))
106+
);
107+
expect(selection.run).toBe(true);
108+
expect(selection.filter).toBe("");
109+
110+
for (const dir of ["unit", "e2e"]) {
111+
const shards = selection.matrix.include
112+
.filter((entry) => entry.dir === dir)
113+
.map((entry) => entry.shard);
114+
expect(shards).not.toHaveLength(0);
115+
116+
const sharded = shards
117+
.flatMap((shard) => listBenchmarks(["--dir", dir, "--shard", shard]))
118+
.sort();
119+
// equality, so a benchmark cannot be dropped or measured twice
120+
expect(sharded).toEqual(listBenchmarks(["--dir", dir]));
121+
}
122+
},
123+
120000
124+
);
125+
60126
for (const file of [...unitFiles, ...e2eFiles]) {
61127
const id = suiteIdOf(file);
62128

‎test/benchmark/lib/cli.mjs‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -62,14 +62,18 @@ for await (const file of fs.glob(
6262

6363
const files = found.sort().filter((_, i) => i % shardCount === shardIndex - 1);
6464

65+
const list = values.list;
66+
6567
const summary = await runSuites(
6668
files.map((file) => path.join(benchmarkRoot, file)),
67-
{ filter, negativeFilter, smoke, maxRme }
69+
{ filter, negativeFilter, smoke, list, maxRme }
6870
);
6971

70-
console.log(
71-
`\n${summary.results.length} benchmark(s) completed, ${summary.failures.length} failed`
72-
);
72+
if (!list) {
73+
console.log(
74+
`\n${summary.results.length} benchmark(s) completed, ${summary.failures.length} failed`
75+
);
76+
}
7377

7478
if (summary.failures.length > 0) {
7579
for (const failure of summary.failures) {

‎test/benchmark/lib/runner.mjs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import { Bench, hrtimeNow } from "tinybench";
99
* @property {RegExp=} filter only run benchmarks whose id matches
1010
* @property {RegExp=} negativeFilter skip benchmarks whose id matches
1111
* @property {boolean=} smoke run every benchmark exactly once, to validate not measure
12+
* @property {boolean=} list print the selected benchmark ids instead of running them
1213
* @property {number=} maxRme relative margin of error warning threshold
1314
*/
1415

@@ -163,6 +164,13 @@ export async function runSuites(files, options) {
163164
});
164165
if (benches.length === 0) continue;
165166

167+
if (options.list) {
168+
for (const definition of benches) {
169+
console.log(`${suite.name}/${definition.name}`);
170+
}
171+
continue;
172+
}
173+
166174
console.log(`\n${suite.name}`);
167175
try {
168176
await suite.setup?.();

0 commit comments

Comments
 (0)