Skip to content

Commit 7f51940

Browse files
ci: split the simulation benchmarks into shards by measured time
1 parent 90b6205 commit 7f51940

3 files changed

Lines changed: 277 additions & 3 deletions

File tree

‎test/BenchmarkTestCases.benchmark.mjs‎

Lines changed: 48 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -212,6 +212,39 @@ const isRoundRobinMode = () => {
212212
return mode === "memory" || mode === "simulation";
213213
};
214214

215+
/**
216+
* Picks this shard's tasks by their measured seconds in `benchmarkTimings.json`:
217+
* longest first, each to the shard with the least time so far. Round-robin left
218+
* the slowest simulation shard 25 minutes behind the fastest.
219+
* @template T
220+
* @param {T[]} tasks tasks, in the same order on every shard
221+
* @param {(task: T) => string} getId the task's id in the timings
222+
* @param {Record<string, number>} timings measured seconds by task id
223+
* @param {[number, number]} shard shard [part, count]
224+
* @returns {T[]} the tasks this shard runs, in their original order
225+
*/
226+
const pickShardTasksByTime = (tasks, getId, timings, shard) => {
227+
const known = Object.values(timings).sort((a, b) => a - b);
228+
// A task added since the timings were measured counts as a typical one.
229+
const fallback = known.length > 0 ? known[Math.floor(known.length / 2)] : 1;
230+
const weighted = tasks.map((task, index) => ({
231+
index,
232+
seconds: timings[getId(task)] ?? fallback
233+
}));
234+
weighted.sort((a, b) => b.seconds - a.seconds || a.index - b.index);
235+
const loads = Array.from({ length: shard[1] }, () => 0);
236+
/** @type {Set<number>} */
237+
const picked = new Set();
238+
239+
for (const { index, seconds } of weighted) {
240+
const target = loads.indexOf(Math.min(...loads));
241+
loads[target] += seconds;
242+
if (target === shard[0] - 1) picked.add(index);
243+
}
244+
245+
return tasks.filter((_, index) => picked.has(index));
246+
};
247+
215248
class BenchmarkRunner {
216249
constructor() {
217250
/** @type {Scenario[]} */
@@ -706,9 +739,21 @@ class BenchmarkRunner {
706739
const splitTasks = await this.splitUnitTasks(benchmarkTasks);
707740

708741
await this.runInWorkers(
709-
isRoundRobinMode()
710-
? splitTasks.filter((_, index) => index % shard[1] === shard[0] - 1)
711-
: splitTasks
742+
getCodspeedRunnerMode() === "simulation"
743+
? pickShardTasksByTime(
744+
splitTasks,
745+
(task) => task.id,
746+
JSON.parse(
747+
await fs.readFile(
748+
path.join(__dirname, "benchmarkTimings.json"),
749+
"utf8"
750+
)
751+
),
752+
shard
753+
)
754+
: isRoundRobinMode()
755+
? splitTasks.filter((_, index) => index % shard[1] === shard[0] - 1)
756+
: splitTasks
712757
);
713758
}
714759
}

‎test/benchmarkTimings.json‎

Lines changed: 228 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,228 @@
1+
{
2+
"asset-modules-bytes-mode-development": 28,
3+
"asset-modules-bytes-mode-development-rebuild": 21,
4+
"asset-modules-bytes-mode-production": 57,
5+
"asset-modules-inline-mode-development": 18,
6+
"asset-modules-inline-mode-development-rebuild": 22,
7+
"asset-modules-inline-mode-production": 77,
8+
"asset-modules-resource-mode-development": 32,
9+
"asset-modules-resource-mode-development-rebuild": 24,
10+
"asset-modules-resource-mode-production": 76,
11+
"asset-modules-source-mode-development": 20,
12+
"asset-modules-source-mode-development-rebuild": 20,
13+
"asset-modules-source-mode-production": 54,
14+
"cache-filesystem-cold-mode-development": 36,
15+
"cache-filesystem-cold-mode-development-rebuild": 24,
16+
"cache-filesystem-cold-mode-production": 182,
17+
"cache-filesystem-mode-development": 83,
18+
"cache-filesystem-mode-development-rebuild": 39,
19+
"cache-filesystem-mode-production": 63,
20+
"common-libs-mode-development": 208,
21+
"common-libs-mode-development-rebuild": 57,
22+
"common-libs-mode-production": 220,
23+
"concatenate-modules-mode-development": 24,
24+
"concatenate-modules-mode-development-rebuild": 26,
25+
"concatenate-modules-mode-production": 52,
26+
"context-commonjs-mode-development": 30,
27+
"context-commonjs-mode-development-rebuild": 24,
28+
"context-commonjs-mode-production": 68,
29+
"context-esm-mode-development": 20,
30+
"context-esm-mode-development-rebuild": 24,
31+
"context-esm-mode-production": 221,
32+
"css-modules-mode-development": 32,
33+
"css-modules-mode-development-rebuild": 24,
34+
"css-modules-mode-production": 146,
35+
"css-parser-unit: unit benchmark \"css-parser-unit\", parseABlocksContents (big)": 17,
36+
"css-parser-unit: unit benchmark \"css-parser-unit\", parseACommaSeparatedListOfComponentValues (big)": 7,
37+
"css-parser-unit: unit benchmark \"css-parser-unit\", parseAComponentValue (big)": 13,
38+
"css-parser-unit: unit benchmark \"css-parser-unit\", parseADeclaration (big)": 12,
39+
"css-parser-unit: unit benchmark \"css-parser-unit\", parseAListOfComponentValues (big)": 12,
40+
"css-parser-unit: unit benchmark \"css-parser-unit\", parseARule (big)": 16,
41+
"css-parser-unit: unit benchmark \"css-parser-unit\", parseAStylesheet (expanded)": 117,
42+
"css-parser-unit: unit benchmark \"css-parser-unit\", parseAStylesheet (min)": 50,
43+
"css-parser-unit: unit benchmark \"css-parser-unit\", parseAStylesheetsContents (min)": 78,
44+
"css-parser-unit: unit benchmark \"css-parser-unit\", process (expanded, no visitors)": 128,
45+
"css-parser-unit: unit benchmark \"css-parser-unit\", process (min, Decl+Url visitors)": 86,
46+
"css-parser-unit: unit benchmark \"css-parser-unit\", process (min, no visitors)": 101,
47+
"css-parser-unit: unit benchmark \"css-parser-unit\", readToken (expanded)": 50,
48+
"css-parser-unit: unit benchmark \"css-parser-unit\", readToken (min)": 22,
49+
"css-printer-unit: unit benchmark \"css-printer-unit\", beautify (buffered blocks)": 50,
50+
"css-printer-unit: unit benchmark \"css-printer-unit\", beautify (deep nesting)": 15,
51+
"css-printer-unit: unit benchmark \"css-printer-unit\", beautify (dropped empty blocks)": 46,
52+
"css-printer-unit: unit benchmark \"css-printer-unit\", beautify (flat rules)": 34,
53+
"css-printer-unit: unit benchmark \"css-printer-unit\", beautify (retracted declarations)": 28,
54+
"css-printer-unit: unit benchmark \"css-printer-unit\", beautify (streamed block)": 30,
55+
"css-printer-unit: unit benchmark \"css-printer-unit\", beautify (tailwind)": 186,
56+
"css-printer-unit: unit benchmark \"css-printer-unit\", minify (buffered blocks)": 46,
57+
"css-printer-unit: unit benchmark \"css-printer-unit\", minify (deep nesting)": 26,
58+
"css-printer-unit: unit benchmark \"css-printer-unit\", minify (dropped empty blocks)": 50,
59+
"css-printer-unit: unit benchmark \"css-printer-unit\", minify (flat rules)": 43,
60+
"css-printer-unit: unit benchmark \"css-printer-unit\", minify (retracted declarations)": 46,
61+
"css-printer-unit: unit benchmark \"css-printer-unit\", minify (streamed block)": 62,
62+
"css-printer-unit: unit benchmark \"css-printer-unit\", minify (tailwind)": 200,
63+
"css-printer-unit: unit benchmark \"css-printer-unit\", walk (buffered blocks)": 25,
64+
"css-printer-unit: unit benchmark \"css-printer-unit\", walk (deep nesting)": 8,
65+
"css-printer-unit: unit benchmark \"css-printer-unit\", walk (dropped empty blocks)": 20,
66+
"css-printer-unit: unit benchmark \"css-printer-unit\", walk (flat rules)": 26,
67+
"css-printer-unit: unit benchmark \"css-printer-unit\", walk (retracted declarations)": 22,
68+
"css-printer-unit: unit benchmark \"css-printer-unit\", walk (streamed block)": 24,
69+
"css-printer-unit: unit benchmark \"css-printer-unit\", walk (tailwind)": 85,
70+
"devtool-eval-mode-development": 26,
71+
"devtool-eval-mode-development-rebuild": 15,
72+
"devtool-eval-mode-production": 93,
73+
"devtool-eval-source-map-mode-development": 31,
74+
"devtool-eval-source-map-mode-development-rebuild": 24,
75+
"devtool-eval-source-map-mode-production": 104,
76+
"devtool-source-map-mode-development": 30,
77+
"devtool-source-map-mode-development-rebuild": 14,
78+
"devtool-source-map-mode-production": 142,
79+
"extract-source-map-mode-development": 20,
80+
"extract-source-map-mode-development-rebuild": 16,
81+
"extract-source-map-mode-production": 46,
82+
"future-defaults-mode-development": 31,
83+
"future-defaults-mode-development-rebuild": 16,
84+
"future-defaults-mode-production": 229,
85+
"hmr-mode-development": 35,
86+
"hmr-mode-development-rebuild": 24,
87+
"hmr-mode-production": 215,
88+
"html-many-chunks-mode-development": 32,
89+
"html-many-chunks-mode-development-rebuild": 13,
90+
"html-many-chunks-mode-production": 38,
91+
"html-parser-unit: unit benchmark \"html-parser-unit\", parseHtml": 138,
92+
"html-parser-unit: unit benchmark \"html-parser-unit\", parseHtml (fragment, tr context)": 26,
93+
"html-parser-unit: unit benchmark \"html-parser-unit\", parseHtml (skip text)": 148,
94+
"html-parser-unit: unit benchmark \"html-parser-unit\", process (Element+Comment visitors)": 100,
95+
"html-parser-unit: unit benchmark \"html-parser-unit\", process (no visitors)": 134,
96+
"html-parser-unit: unit benchmark \"html-parser-unit\", tokenize": 42,
97+
"html-printer-unit: unit benchmark \"html-printer-unit\", beautify (composed subtrees)": 26,
98+
"html-printer-unit: unit benchmark \"html-printer-unit\", beautify (deep nesting)": 21,
99+
"html-printer-unit: unit benchmark \"html-printer-unit\", beautify (document)": 66,
100+
"html-printer-unit: unit benchmark \"html-printer-unit\", beautify (flat siblings)": 12,
101+
"html-printer-unit: unit benchmark \"html-printer-unit\", beautify (fragment, tr context)": 20,
102+
"html-printer-unit: unit benchmark \"html-printer-unit\", beautify (implied start tags)": 10,
103+
"html-printer-unit: unit benchmark \"html-printer-unit\", beautify (nested lists)": 34,
104+
"html-printer-unit: unit benchmark \"html-printer-unit\", beautify (sparse inline style)": 7,
105+
"html-printer-unit: unit benchmark \"html-printer-unit\", beautify (wide table)": 44,
106+
"html-printer-unit: unit benchmark \"html-printer-unit\", minify (composed subtrees)": 28,
107+
"html-printer-unit: unit benchmark \"html-printer-unit\", minify (deep nesting)": 46,
108+
"html-printer-unit: unit benchmark \"html-printer-unit\", minify (document)": 78,
109+
"html-printer-unit: unit benchmark \"html-printer-unit\", minify (flat siblings)": 28,
110+
"html-printer-unit: unit benchmark \"html-printer-unit\", minify (fragment, tr context)": 40,
111+
"html-printer-unit: unit benchmark \"html-printer-unit\", minify (implied start tags)": 18,
112+
"html-printer-unit: unit benchmark \"html-printer-unit\", minify (nested lists)": 41,
113+
"html-printer-unit: unit benchmark \"html-printer-unit\", minify (sparse inline style)": 7,
114+
"html-printer-unit: unit benchmark \"html-printer-unit\", minify (wide table)": 54,
115+
"html-printer-unit: unit benchmark \"html-printer-unit\", walk (composed subtrees)": 18,
116+
"html-printer-unit: unit benchmark \"html-printer-unit\", walk (deep nesting)": 24,
117+
"html-printer-unit: unit benchmark \"html-printer-unit\", walk (document)": 44,
118+
"html-printer-unit: unit benchmark \"html-printer-unit\", walk (flat siblings)": 15,
119+
"html-printer-unit: unit benchmark \"html-printer-unit\", walk (fragment, tr context)": 22,
120+
"html-printer-unit: unit benchmark \"html-printer-unit\", walk (implied start tags)": 12,
121+
"html-printer-unit: unit benchmark \"html-printer-unit\", walk (nested lists)": 24,
122+
"html-printer-unit: unit benchmark \"html-printer-unit\", walk (sparse inline style)": 6,
123+
"html-printer-unit: unit benchmark \"html-printer-unit\", walk (wide table)": 30,
124+
"import-meta-glob-mode-development": 54,
125+
"import-meta-glob-mode-development-rebuild": 20,
126+
"import-meta-glob-mode-production": 224,
127+
"js-minifier-unit: unit benchmark \"js-minifier-unit\", lodash": 204,
128+
"js-minimizer-plugin-unit: unit benchmark \"js-minimizer-plugin-unit\", minify entry by module path": 8,
129+
"js-minimizer-plugin-unit: unit benchmark \"js-minimizer-plugin-unit\", transform entry by function source": 8,
130+
"js-parser-unit: unit benchmark \"js-parser-unit\", mode 'parse'": 398,
131+
"js-parser-unit: unit benchmark \"js-parser-unit\", source 'lodash-es'": 6,
132+
"js-parser-unit: unit benchmark \"js-parser-unit\", source 'lodash.js'": 24,
133+
"js-parser-unit: unit benchmark \"js-parser-unit\", source 'react-dom.development.js'": 5,
134+
"js-parser-unit: unit benchmark \"js-parser-unit\", source 'react.development.js'": 9,
135+
"js-parser-unit: unit benchmark \"js-parser-unit\", source 'three.module.js'": 42,
136+
"js-parser-unit: unit benchmark \"js-parser-unit\", source 'three.module.min.js'": 36,
137+
"js-parser-unit: unit benchmark \"js-parser-unit\", sourceType 'auto'": 460,
138+
"js-parser-unit: unit benchmark \"js-parser-unit\", sourceType 'module'": 323,
139+
"js-parser-unit: unit benchmark \"js-parser-unit\", sourceType 'script'": 561,
140+
"js-parser-unit: unit benchmark \"js-parser-unit\", tokenize three.module.js": 24,
141+
"js-parser-unit: unit benchmark \"js-parser-unit\", tokenize three.module.min.js": 18,
142+
"js-parser-unit: unit benchmark \"js-parser-unit\", tokenize typescript": 274,
143+
"js-printer-unit: unit benchmark \"js-printer-unit\", mangle (lodash.js)": 74,
144+
"js-printer-unit: unit benchmark \"js-printer-unit\", mangle (three.module.js)": 146,
145+
"js-printer-unit: unit benchmark \"js-printer-unit\", minify (lodash.js)": 154,
146+
"js-printer-unit: unit benchmark \"js-printer-unit\", minify (three.module.js)": 257,
147+
"js-printer-unit: unit benchmark \"js-printer-unit\", print (lodash.js)": 32,
148+
"js-printer-unit: unit benchmark \"js-printer-unit\", print (three.module.js)": 102,
149+
"js-scope-analyzer-unit: unit benchmark \"js-scope-analyzer-unit\", analyze 'lodash.js'": 30,
150+
"js-scope-analyzer-unit: unit benchmark \"js-scope-analyzer-unit\", analyze 'three.module.js'": 37,
151+
"js-scope-analyzer-unit: unit benchmark \"js-scope-analyzer-unit\", analyze 'typescript.js'": 44,
152+
"js-scope-analyzer-unit: unit benchmark \"js-scope-analyzer-unit\", parse 'lodash.js'": 35,
153+
"js-scope-analyzer-unit: unit benchmark \"js-scope-analyzer-unit\", parse 'three.module.js'": 64,
154+
"js-scope-analyzer-unit: unit benchmark \"js-scope-analyzer-unit\", parse 'typescript.js'": 492,
155+
"js-scope-analyzer-unit: unit benchmark \"js-scope-analyzer-unit\", parse+analyze 'lodash.js'": 42,
156+
"js-scope-analyzer-unit: unit benchmark \"js-scope-analyzer-unit\", parse+analyze 'three.module.js'": 66,
157+
"js-scope-analyzer-unit: unit benchmark \"js-scope-analyzer-unit\", parse+analyze 'typescript.js'": 307,
158+
"json-modules-mode-development": 28,
159+
"json-modules-mode-development-rebuild": 23,
160+
"json-modules-mode-production": 52,
161+
"loader-babel-mode-development": 36,
162+
"loader-babel-mode-development-rebuild": 29,
163+
"loader-babel-mode-production": 97,
164+
"lodash-mode-development": 44,
165+
"lodash-mode-development-rebuild": 22,
166+
"lodash-mode-production": 383,
167+
"many-chunks-commonjs-mode-development": 22,
168+
"many-chunks-commonjs-mode-development-rebuild": 24,
169+
"many-chunks-commonjs-mode-production": 144,
170+
"many-chunks-esm-mode-development": 35,
171+
"many-chunks-esm-mode-development-rebuild": 22,
172+
"many-chunks-esm-mode-production": 160,
173+
"many-modules-commonjs-mode-development": 24,
174+
"many-modules-commonjs-mode-development-rebuild": 25,
175+
"many-modules-commonjs-mode-production": 142,
176+
"many-modules-esm-mode-development": 38,
177+
"many-modules-esm-mode-development-rebuild": 27,
178+
"many-modules-esm-mode-production": 160,
179+
"many-modules-interop-runtime-mode-development": 25,
180+
"many-modules-interop-runtime-mode-production": 110,
181+
"module-unsafe-cache-mode-development": 33,
182+
"module-unsafe-cache-mode-development-rebuild": 24,
183+
"module-unsafe-cache-mode-production": 172,
184+
"no-concatenation-mode-development": 36,
185+
"no-concatenation-mode-development-rebuild": 16,
186+
"no-concatenation-mode-production": 172,
187+
"no-exports-analysis-mode-development": 33,
188+
"no-exports-analysis-mode-development-rebuild": 27,
189+
"no-exports-analysis-mode-production": 183,
190+
"no-minimize-mode-development": 36,
191+
"no-minimize-mode-development-rebuild": 14,
192+
"no-minimize-mode-production": 45,
193+
"react-mode-development": 30,
194+
"react-mode-development-rebuild": 18,
195+
"react-mode-production": 86,
196+
"resolve-unsafe-cache-mode-development": 37,
197+
"resolve-unsafe-cache-mode-development-rebuild": 16,
198+
"resolve-unsafe-cache-mode-production": 174,
199+
"side-effects-reexport-mode-development": 20,
200+
"side-effects-reexport-mode-development-rebuild": 20,
201+
"side-effects-reexport-mode-production": 44,
202+
"three-long-mode-development": 130,
203+
"three-long-mode-development-rebuild": 43,
204+
"three-long-mode-production": 186,
205+
"unsafe-cache-mode-development": 34,
206+
"unsafe-cache-mode-development-rebuild": 14,
207+
"unsafe-cache-mode-production": 178,
208+
"wasm-modules-async-mode-development": 34,
209+
"wasm-modules-async-mode-development-rebuild": 26,
210+
"wasm-modules-async-mode-production": 104,
211+
"wasm-modules-sync-mode-development": 48,
212+
"wasm-modules-sync-mode-development-rebuild": 16,
213+
"wasm-modules-sync-mode-production": 132,
214+
"xml-parser-unit: unit benchmark \"xml-parser-unit\", parseHtml": 32,
215+
"xml-parser-unit: unit benchmark \"xml-parser-unit\", parseHtml (skip text)": 37,
216+
"xml-parser-unit: unit benchmark \"xml-parser-unit\", process (Element+Comment visitors)": 34,
217+
"xml-parser-unit: unit benchmark \"xml-parser-unit\", process (no visitors)": 41,
218+
"xml-parser-unit: unit benchmark \"xml-parser-unit\", tokenize": 11,
219+
"xml-printer-unit: unit benchmark \"xml-printer-unit\", beautify (deep nesting)": 18,
220+
"xml-printer-unit: unit benchmark \"xml-printer-unit\", beautify (document)": 19,
221+
"xml-printer-unit: unit benchmark \"xml-printer-unit\", beautify (flat siblings)": 14,
222+
"xml-printer-unit: unit benchmark \"xml-printer-unit\", minify (deep nesting)": 24,
223+
"xml-printer-unit: unit benchmark \"xml-printer-unit\", minify (document)": 28,
224+
"xml-printer-unit: unit benchmark \"xml-printer-unit\", minify (flat siblings)": 14,
225+
"xml-printer-unit: unit benchmark \"xml-printer-unit\", walk (deep nesting)": 17,
226+
"xml-printer-unit: unit benchmark \"xml-printer-unit\", walk (document)": 18,
227+
"xml-printer-unit: unit benchmark \"xml-printer-unit\", walk (flat siblings)": 16
228+
}

‎test/unitCases/javascript/parser.unittest.js‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
/** @typedef {typeof import("../../../lib/javascript/syntax-parser")} ParserExports */
1010
/** @typedef {Partial<import("../../../lib/javascript/syntax-parser").Options>} PortOptions */
1111
/** @typedef {import("../../../lib/javascript/syntax-parser").Node} EstreeNode */
12+
/** @typedef {import("../../../lib/javascript/syntax-parser").ParserOptions} ParserOptions */
1213
/**
1314
* A parsed node whose children may already stand as the ids they were written as.
1415
* @typedef {{ type: string, [key: string]: unknown }} WrittenNode

0 commit comments

Comments
 (0)