Skip to content

Commit b97505b

Browse files
alexander-akaitalexander-akait
committed
test(benchmark): split the e2e suites out of this PR
The harness, the unit benches and the measurement CI each raise a different review question. Keep this PR to the harness and the unit suites; the e2e benches, the webpack lifecycle helpers and project generators only they use, and their selection rules move to their own PR.
1 parent dcd704e commit b97505b

18 files changed

Lines changed: 26 additions & 1252 deletions

File tree

‎.github/scripts/select-benchmarks.mjs‎

Lines changed: 8 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -4,80 +4,29 @@ import { execFileSync } from "child_process";
44
import fs from "fs";
55

66
const fullMatrix = {
7-
include: [
8-
{ dir: "unit", shard: "1/1" },
9-
{ dir: "e2e", shard: "1/3" },
10-
{ dir: "e2e", shard: "2/3" },
11-
{ dir: "e2e", shard: "3/3" }
12-
]
7+
include: [{ dir: "unit", shard: "1/1" }]
138
};
149

1510
const benchmarkFiles = execFileSync(
1611
"git",
17-
["ls-files", "-z", "test/benchmark/unit", "test/benchmark/e2e"],
12+
["ls-files", "-z", "test/benchmark/unit"],
1813
{ encoding: "utf8" }
1914
)
2015
.split("\0")
2116
.filter((file) => file.endsWith(".bench.mjs"))
2217
.sort();
2318

2419
const suiteByFile = new Map(
25-
benchmarkFiles.map((file) => {
26-
if (file.startsWith("test/benchmark/e2e/")) {
27-
return [file, file.split("/").slice(2, 4).join("/")];
28-
}
29-
return [
30-
file,
31-
file
32-
.slice("test/benchmark/".length, -".bench.mjs".length)
33-
.replaceAll("\\", "/")
34-
];
35-
})
20+
benchmarkFiles.map((file) => [
21+
file,
22+
file
23+
.slice("test/benchmark/".length, -".bench.mjs".length)
24+
.replaceAll("\\", "/")
25+
])
3626
);
3727
const fileBySuite = new Map(
3828
[...suiteByFile].map(([file, suite]) => [suite, file])
3929
);
40-
const e2eFiles = benchmarkFiles.filter((file) =>
41-
file.startsWith("test/benchmark/e2e/")
42-
);
43-
44-
const e2eRules = [
45-
["e2e/asset-modules", [/^lib\/asset\//, /^lib\/AssetModulesPlugin\.js$/]],
46-
[
47-
"e2e/code-splitting",
48-
[
49-
/^lib\/optimize\//,
50-
/^lib\/(?:Chunk|ChunkGraph|ChunkGroup|Entrypoint|buildChunkGraph)\.js$/
51-
]
52-
],
53-
["e2e/css-modules", [/^lib\/css\//]],
54-
[
55-
"e2e/filesystem-cache",
56-
[
57-
/^lib\/cache\//,
58-
/^lib\/serialization\//,
59-
/^lib\/(?:Cache|CacheFacade)\.js$/,
60-
/^lib\/util\/serialization\.js$/
61-
]
62-
],
63-
["e2e/json-modules", [/^lib\/json\//]],
64-
[
65-
"e2e/rebuild",
66-
[
67-
/^hot\//,
68-
/^lib\/hmr\//,
69-
/^lib\/cache\//,
70-
/^lib\/(?:Compilation|Compiler|FileSystemInfo|NormalModule|Watching)\.js$/
71-
]
72-
],
73-
[
74-
"e2e/source-map",
75-
[
76-
/^lib\/(?:EvalSourceMapDevToolPlugin|SourceMapDevToolModuleOptionsPlugin|SourceMapDevToolPlugin)\.js$/,
77-
/^lib\/util\/extractSourceMap\.js$/
78-
]
79-
]
80-
];
8130

8231
const escapeRegExp = (value) => value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
8332

@@ -123,18 +72,9 @@ const recommendedSelection = (changedFiles) => {
12372
addSuite(`unit/${file.slice("lib/".length, -".js".length)}`);
12473
}
12574

126-
if (/^(lib|schemas)\//.test(file)) {
127-
addSuite("e2e/many-modules-commonjs");
128-
addSuite("e2e/many-modules-esm");
129-
}
130-
13175
if (file.startsWith("assembly/")) {
13276
addSuite("unit/util/createHash");
13377
}
134-
135-
for (const [suite, patterns] of e2eRules) {
136-
if (patterns.some((pattern) => pattern.test(file))) addSuite(suite);
137-
}
13878
}
13979

14080
if (suites.size === 0) return skippedSelection();
@@ -145,17 +85,6 @@ const recommendedSelection = (changedFiles) => {
14585
include.push({ dir: "unit", shard: "1/1" });
14686
}
14787

148-
const e2eShards = new Set();
149-
for (const suite of sortedSuites) {
150-
if (!suite.startsWith("e2e/")) continue;
151-
const file = fileBySuite.get(suite);
152-
const index = file === undefined ? -1 : e2eFiles.indexOf(file);
153-
if (index >= 0) e2eShards.add((index % 3) + 1);
154-
}
155-
for (const shard of [...e2eShards].sort()) {
156-
include.push({ dir: "e2e", shard: `${shard}/3` });
157-
}
158-
15988
return {
16089
run: true,
16190
filter: `^(?:${sortedSuites.map(escapeRegExp).join("|")})(?:/|$)`,

‎.github/workflows/test.yml‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -162,10 +162,6 @@ jobs:
162162
163163
- run: yarn cover:unit --ci --cacheDirectory .jest-cache
164164

165-
# test/Benchmarks.unittest.js smoke-runs the unit suites; the e2e ones
166-
# build real projects, so they run here instead of under jest.
167-
- run: yarn benchmark:e2e --smoke
168-
169165
- name: Codecov
170166
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
171167
with:

‎AGENTS.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,7 @@ The directory listings below are the canonical map of the repository. **Whenever
103103
**Tests** — see [TESTING_DOCS.md](TESTING_DOCS.md) for directory structure, naming, and how to run a single case.
104104

105105
- `test/` — All test suites (`cases/`, `configCases/`, `watchCases/`, `hotCases/`, `statsCases/`, `typesCases/`, `test262-cases/`, `html5lib-tests/`, `wpt/`, `css-parsing-tests/`, `benchmarkCases/`, `memoryLimitCases/`, etc.). `RoundTripConfigCases` re-bundles the output of `configCases` marked with a `roundTrip.js` file. `wpt/` is web-platform-tests, checked out one commit deep by the `html5lib` job alone — it is where the HTML tree-construction corpus lives since html5lib-tests dropped it.
106-
- `test/benchmark/` — Non-comparative benchmark harness (`lib/`), unit benchmarks (`unit/`) and full-build benchmarks (`e2e/`); see `test/benchmark/README.md`.
106+
- `test/benchmark/` — Non-comparative benchmark harness (`lib/`) and unit benchmarks (`unit/`); see `test/benchmark/README.md`.
107107

108108
**Examples & changesets**
109109

‎TESTING_DOCS.md‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,9 @@ This document explains the structure of the `test/` directory in the Webpack pro
2323

2424
### 2b. `benchmark/`
2525

26-
- **Purpose**: Non-comparative, CodSpeed-integrated benchmarks: `unit/` for webpack internals, `e2e/` for full builds, `lib/` for the harness itself.
27-
- **Usage**: `yarn benchmark:unit`, `yarn benchmark:e2e` or `yarn benchmark:suite`; see [test/benchmark/README.md](test/benchmark/README.md).
28-
- **Kept working by**: `test/Benchmarks.unittest.js`, which runs every `unit/` suite once (`--smoke`) as part of `yarn test:unit`, and the `yarn benchmark:e2e --smoke` step in the `unit` CI job. Measurement itself only runs on schedule or behind a label, so without these a suite broken by a `lib/` change would stay unnoticed until then.
26+
- **Purpose**: Non-comparative, CodSpeed-integrated benchmarks: `unit/` for webpack internals, `lib/` for the harness itself.
27+
- **Usage**: `yarn benchmark:unit` or `yarn benchmark:suite`; see [test/benchmark/README.md](test/benchmark/README.md).
28+
- **Kept working by**: `test/Benchmarks.unittest.js`, which runs every suite once (`--smoke`) as part of `yarn test:unit`. Measurement itself only runs on schedule or behind a label, so without it a suite broken by a `lib/` change would stay unnoticed until then.
2929

3030
### 3. `cases/`
3131

‎package.json‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,6 @@
5656
"benchmark:html-minifiers": "node tooling/compare-html-minifiers.js",
5757
"benchmark:suite": "node --perf-prof --perf-basic-prof --interpreted-frames-native-stack --allow-natives-syntax ./test/benchmark/lib/cli.mjs",
5858
"benchmark:unit": "yarn benchmark:suite --dir unit",
59-
"benchmark:e2e": "yarn benchmark:suite --dir e2e",
6059
"pretest": "yarn lint",
6160
"test": "yarn test:base",
6261
"test:base": "node --expose-gc --max-old-space-size=4096 --experimental-vm-modules --trace-deprecation node_modules/jest-cli/bin/jest --workerIdleMemoryLimit=512MB --logHeapUsage",

‎test/Benchmarks.unittest.js‎

Lines changed: 11 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -89,11 +89,9 @@ const importDefault = async (file) => {
8989

9090
describe("Benchmarks", () => {
9191
const unitFiles = findBenchmarkFiles(path.join(benchmarkRoot, "unit"));
92-
const e2eFiles = findBenchmarkFiles(path.join(benchmarkRoot, "e2e"));
9392

9493
it("should discover the benchmark suites", () => {
9594
expect(unitFiles).not.toHaveLength(0);
96-
expect(e2eFiles).not.toHaveLength(0);
9795
});
9896

9997
// the scheduled run is the only one that measures everything, and its shard
@@ -107,23 +105,21 @@ describe("Benchmarks", () => {
107105
expect(selection.run).toBe(true);
108106
expect(selection.filter).toBe("");
109107

110-
for (const dir of ["unit", "e2e"]) {
111-
const shards = selection.matrix.include
112-
.filter((entry) => entry.dir === dir)
113-
.map((entry) => entry.shard);
114-
expect(shards).not.toHaveLength(0);
115-
116-
const sharded = shards
117-
.flatMap((shard) => listBenchmarks(["--dir", dir, "--shard", shard]))
118-
.sort();
119-
// equality, so a benchmark cannot be dropped or measured twice
120-
expect(sharded).toEqual(listBenchmarks(["--dir", dir]));
121-
}
108+
const shards = selection.matrix.include
109+
.filter((entry) => entry.dir === "unit")
110+
.map((entry) => entry.shard);
111+
expect(shards).not.toHaveLength(0);
112+
113+
const sharded = shards
114+
.flatMap((shard) => listBenchmarks(["--dir", "unit", "--shard", shard]))
115+
.sort();
116+
// equality, so a benchmark cannot be dropped or measured twice
117+
expect(sharded).toEqual(listBenchmarks(["--dir", "unit"]));
122118
},
123119
120000
124120
);
125121

126-
for (const file of [...unitFiles, ...e2eFiles]) {
122+
for (const file of unitFiles) {
127123
const id = suiteIdOf(file);
128124

129125
itNode(`should name suite ${id} after its location`, async () => {
@@ -134,10 +130,6 @@ describe("Benchmarks", () => {
134130
suite.benches.length
135131
);
136132
});
137-
}
138-
139-
for (const file of unitFiles) {
140-
const id = suiteIdOf(file);
141133

142134
it(`should keep ${id} mirroring a core file`, () => {
143135
const core = path.join(

‎test/benchmark/README.md‎

Lines changed: 3 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,8 @@ wall-clock latency.
77
## Layout
88

99
- `unit/` contains focused core benchmarks and mirrors `lib/`.
10-
- `e2e/` contains full webpack builds grouped by workload.
1110
- `helpers/` contains deterministic fixture generators.
12-
- `lib/` contains suite discovery, execution and webpack lifecycle helpers.
11+
- `lib/` contains suite discovery and execution.
1312

1413
A unit suite has the same relative path and exact basename as its primary core
1514
file. For example:
@@ -30,13 +29,11 @@ should primarily belong to the named core file.
3029

3130
```bash
3231
yarn benchmark:unit
33-
yarn benchmark:e2e
3432
yarn benchmark:suite
3533
yarn benchmark:suite --filter "JavascriptParser"
3634
yarn benchmark:suite --dir unit --smoke
37-
yarn benchmark:suite --dir e2e --shard 1/2
35+
yarn benchmark:suite --dir unit --shard 1/2
3836
yarn benchmark:suite --list
39-
yarn benchmark:suite --json test/js/benchmark-results.json
4037
yarn benchmark:suite --max-rme 5
4138
```
4239

@@ -46,24 +43,6 @@ before a full run when adding or changing suites.
4643
Wall-time runs warn when any benchmark exceeds 15% RME. Use `--max-rme` or
4744
`MAX_RME` to select a different threshold.
4845

49-
## Adding an e2e suite
50-
51-
Use `createBuildScenarios` for each configuration. It creates development and
52-
production builds plus a development rebuild in temporary output directories:
53-
54-
```js
55-
const name = "e2e/many-modules-esm";
56-
57-
export default {
58-
name,
59-
benches: createBuildScenarios({
60-
case: "without-module-concatenation",
61-
entryFile,
62-
config: { entry }
63-
})
64-
};
65-
```
66-
6746
## Adding a unit suite
6847

6948
1. Choose a core file with work substantial enough to regress build time or
@@ -101,6 +80,4 @@ selects affected suites and shards from the changed files. CodSpeed fills
10180
unmeasured benchmarks from the baseline as a
10281
[partial run](https://codspeed.io/docs/features/partial-runs).
10382

104-
Fixtures must not depend on time, ambient files or `Math.random()`. Generated
105-
e2e projects belong in a `generated/` directory, which is ignored by Git and
106-
recreated by the suite.
83+
Fixtures must not depend on time, ambient files or `Math.random()`.

‎test/benchmark/e2e/asset-modules/index.bench.mjs‎

Lines changed: 0 additions & 72 deletions
This file was deleted.

0 commit comments

Comments
 (0)