Skip to content

Commit d74fe17

Browse files
ci: measure CPU benchmarks with CodSpeed walltime (#22764)
Simulation leaves system calls out of the measure, which CodSpeed flagged on 108 benchmarks. Walltime runs on CodSpeed macro runners, one task per fresh worker, and reports tinybench's samples as walltime results. Claude-Session: https://claude.ai/code/session_01M6ivBSByXdivy6s5XG6Q5U
1 parent 2cd72c7 commit d74fe17

4 files changed

Lines changed: 134 additions & 26 deletions

File tree

‎.github/workflows/benchmarks.yml‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,9 @@ jobs:
3434
fail-fast: false
3535
matrix:
3636
shard: [1/4, 2/4, 3/4, 4/4]
37-
runs-on: ubuntu-latest
37+
# Walltime measures real time, system calls included, so it needs CodSpeed's
38+
# dedicated runners: a shared GitHub runner is too noisy to compare against.
39+
runs-on: codspeed-macro
3840
# The harness has no deadline of its own and a watch bench waits on a
3941
# rebuild callback; main's slowest shard is ~55m, so this only trips on a hang.
4042
timeout-minutes: 120
@@ -69,14 +71,11 @@ jobs:
6971
uses: CodSpeedHQ/action@f842b9a3ffe213dbb81768fd7f7eeb4dfcf1c369 # v5.4.2
7072
with:
7173
run: yarn benchmark --ci
72-
mode: "simulation"
74+
mode: "walltime"
7375
env:
7476
LAST_COMMIT: 1
7577
NEGATIVE_FILTER: ${{ github.event_name != 'schedule' && 'on-schedule' || '' }}
7678
SHARD: ${{ matrix.shard }}
77-
# Single libuv thread → deterministic fs-completion order, removing the
78-
# last I/O race that perturbs allocation counts between runs.
79-
UV_THREADPOOL_SIZE: 1
8079
benchmark-memory:
8180
strategy:
8281
fail-fast: false

‎cspell.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -489,6 +489,7 @@
489489
"srcset",
490490
"stackoverflow",
491491
"starstar",
492+
"stdev",
492493
"Stensberg",
493494
"stringable",
494495
"strtok",

‎test/BenchmarkTestCases.benchmark.mjs‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,8 @@ const REV_LIST_REGEXP = /^([a-f0-9]+)\s*([a-f0-9]+)\s*([a-f0-9]+)?\s*$/;
5555
const LAST_COMMIT = typeof process.env.LAST_COMMIT !== "undefined";
5656

5757
const checkV8Flags = () => {
58+
// Walltime measures only in the workers, which get its flags at fork.
59+
if (getCodspeedRunnerMode() === "walltime") return;
5860
const requiredFlags = getV8Flags().filter(
5961
(flag) => !flag.startsWith("--max-old-space-size")
6062
);
@@ -521,6 +523,9 @@ class BenchmarkRunner {
521523
// Memory mode measures each task alone in a fresh process, so its counts
522524
// don't depend on which tasks share a shard and match a filtered local run.
523525
const isolated = getCodspeedRunnerMode() === "memory";
526+
// Walltime times every task alone, so no other build competes for the CPU,
527+
// and in a fresh process, so earlier tasks leave no JIT or heap state.
528+
const walltime = getCodspeedRunnerMode() === "walltime";
524529

525530
// Bound the pool by RAM, not just cores: a Valgrind build peaks near 11 GiB,
526531
// so 16 GiB fits one worker; bigger runners auto-scale.
@@ -534,7 +539,8 @@ class BenchmarkRunner {
534539

535540
// CodSpeed's runner tracks one benchmark at a time: concurrent workers
536541
// interleave start/stop markers and the run reports no results.
537-
const numWorkers = isolated ? 1 : Math.min(cpuWorkers, memWorkers);
542+
const numWorkers =
543+
isolated || walltime ? 1 : Math.min(cpuWorkers, memWorkers);
538544
// Memory mode counts native allocations, so V8's own work must stay out of them:
539545
// Maglev compiles despite `--no-opt`, helper threads split GC work, and a major
540546
// GC starts on a timer or falls inside the sample by a few KB of live heap.
@@ -556,7 +562,8 @@ class BenchmarkRunner {
556562
// Valgrind memory accumulates across builds and frees only on exit, so
557563
// recycle the worker after each task (`0` = always restart) to cap peak
558564
// at one build's footprint; otherwise a shard OOMs mid-run (exit 143).
559-
idleMemoryLimit: underValgrind || isolated ? 0 : undefined,
565+
idleMemoryLimit:
566+
underValgrind || isolated || walltime ? 0 : undefined,
560567
// Forward the V8 flags CodSpeed needs (seeds, --no-opt, …) so the
561568
// child processes measure under the same deterministic conditions.
562569
forkOptions: {

‎test/harness/benchmark/benchmark.worker.mjs‎

Lines changed: 120 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -8,11 +8,17 @@ import { fileURLToPath, pathToFileURL } from "url";
88
import vm from "vm";
99
import {
1010
InstrumentHooks,
11+
MARKER_TYPE_BENCHMARK_END,
12+
MARKER_TYPE_BENCHMARK_START,
13+
calculateQuantiles,
1114
getCodspeedRunnerMode,
1215
getGitDir,
1316
mongoMeasurement,
17+
msToNs,
18+
msToS,
1419
setupCore,
15-
teardownCore
20+
teardownCore,
21+
writeWalltimeResults
1622
} from "@codspeed/core";
1723
import { Bench, hrtimeNow } from "tinybench";
1824

@@ -25,6 +31,7 @@ import { Bench, hrtimeNow } from "tinybench";
2531
* } from "tinybench"
2632
*/
2733
/** @typedef {TinybenchTask & { collectBy?: string }} Task */
34+
/** @import { Benchmark } from "@codspeed/core" */
2835

2936
/** @import Webpack, { Configuration, Stats, Watching } from "../../.." */
3037
/**
@@ -262,6 +269,23 @@ function getTaskUri(bench, taskName, callingFile) {
262269
return uriMap?.get(taskName)?.uri || `${callingFile}::${taskName}`;
263270
}
264271

272+
/**
273+
* @param {Fn} fn function
274+
* @param {boolean} isAsync true if async function, otherwise false
275+
* @returns {() => void} function wrapped in codspeed root frame
276+
*/
277+
const wrapFunctionWithFrame = (fn, isAsync) => {
278+
if (isAsync) {
279+
return async function __codspeed_root_frame__() {
280+
await fn();
281+
};
282+
}
283+
284+
return function __codspeed_root_frame__() {
285+
fn();
286+
};
287+
};
288+
265289
/**
266290
* @param {Bench} bench bench
267291
* @returns {Promise<Bench>} modifier bench
@@ -288,6 +312,15 @@ const withCodSpeed = async (bench) => {
288312
if (releaseClosuresAfterMeasure) {
289313
return rawAdd.bind(bench)(name, NO_OP_FN, {});
290314
}
315+
if (codspeedRunnerMode === "walltime") {
316+
// Walltime samples outside this frame are not attributed to the bench.
317+
const isAsync = fn.constructor.name === "AsyncFunction";
318+
return rawAdd.bind(bench)(
319+
name,
320+
wrapFunctionWithFrame(fn, isAsync),
321+
options
322+
);
323+
}
291324
return rawAdd.bind(bench)(name, fn, options);
292325
};
293326
const callingFile = getCallingFile();
@@ -311,23 +344,6 @@ const withCodSpeed = async (bench) => {
311344
return bench.tasks;
312345
};
313346

314-
/**
315-
* @param {Fn} fn function
316-
* @param {boolean} isAsync true if async function, otherwise false
317-
* @returns {() => void} function wrapped in codspeed root frame
318-
*/
319-
const wrapFunctionWithFrame = (fn, isAsync) => {
320-
if (isAsync) {
321-
return async function __codspeed_root_frame__() {
322-
await fn();
323-
};
324-
}
325-
326-
return function __codspeed_root_frame__() {
327-
fn();
328-
};
329-
};
330-
331347
/**
332348
* @param {string} uri URI
333349
* @param {string} status status
@@ -653,12 +669,95 @@ const withCodSpeed = async (bench) => {
653669
return finalizeSyncRun();
654670
};
655671
} else if (codspeedRunnerMode === "walltime") {
656-
// We don't need it
672+
/** @type {Benchmark[]} */
673+
const codspeedBenchmarks = [];
674+
675+
bench.run = async () => {
676+
setupCore();
677+
console.log(
678+
`[CodSpeed] running with @codspeed/tinybench (walltime mode, ${bench.warmupIterations} warmup iterations)`
679+
);
680+
681+
for (const task of bench.tasks) {
682+
const uri = getTaskUri(bench, task.name, callingFile);
683+
684+
await task.warmup();
685+
InstrumentHooks.startBenchmark();
686+
const runStart = InstrumentHooks.currentTimestamp();
687+
await task.run();
688+
const runEnd = InstrumentHooks.currentTimestamp();
689+
InstrumentHooks.stopBenchmark();
690+
InstrumentHooks.setExecutedBenchmark(process.pid, uri);
691+
InstrumentHooks.addMarker(
692+
process.pid,
693+
MARKER_TYPE_BENCHMARK_START,
694+
runStart
695+
);
696+
InstrumentHooks.addMarker(
697+
process.pid,
698+
MARKER_TYPE_BENCHMARK_END,
699+
runEnd
700+
);
701+
702+
codspeedBenchmarks.push(toWalltimeBenchmark(bench, task, uri));
703+
console.log(`[CodSpeed] Collected walltime data for ${uri}`);
704+
}
705+
706+
writeWalltimeResults(codspeedBenchmarks, true);
707+
teardownCore();
708+
console.log(`[CodSpeed] Done running ${bench.tasks.length} benches.`);
709+
return bench.tasks;
710+
};
657711
}
658712

659713
return bench;
660714
};
661715

716+
/**
717+
* Convert a finished tinybench task into CodSpeed's walltime result entry.
718+
* Tinybench runs one round, reported as one round of one iteration per sample.
719+
* @param {Bench} bench bench
720+
* @param {TinybenchTask} task finished task, run with `retainSamples`
721+
* @param {string} uri task URI
722+
* @returns {Benchmark} walltime result entry
723+
*/
724+
function toWalltimeBenchmark(bench, task, uri) {
725+
const result = task.result;
726+
727+
if (!result || result.state !== "completed") {
728+
throw new Error(`No walltime result for ${uri}`);
729+
}
730+
731+
const { min, max, mean, sd, samples } = result.latency;
732+
const sortedTimesNs = /** @type {number[]} */ (samples)
733+
.map(msToNs)
734+
.sort((a, b) => a - b);
735+
const meanNs = msToNs(mean);
736+
const stdevNs = msToNs(sd);
737+
738+
return {
739+
name: task.name,
740+
uri,
741+
config: {
742+
max_rounds: bench.iterations,
743+
max_time_ns: msToNs(bench.time),
744+
min_round_time_ns: null,
745+
warmup_time_ns: msToNs(bench.warmupTime)
746+
},
747+
stats: {
748+
min_ns: msToNs(min),
749+
max_ns: msToNs(max),
750+
mean_ns: meanNs,
751+
stdev_ns: stdevNs,
752+
...calculateQuantiles({ sortedTimesNs, stdevNs, meanNs }),
753+
total_time: msToS(result.totalTime),
754+
iter_per_round: 1,
755+
rounds: sortedTimesNs.length,
756+
warmup_iters: bench.warmupIterations
757+
}
758+
};
759+
}
760+
662761
/**
663762
* @param {number} n number of runs
664763
* @returns {number} distribution
@@ -1237,6 +1336,8 @@ function createBenchInstance() {
12371336
warmup: true,
12381337
warmupIterations: 2,
12391338
iterations: 8,
1339+
// Walltime reports quantiles, computed from the raw samples.
1340+
retainSamples: codspeedRunnerMode === "walltime",
12401341
setup(task, mode) {
12411342
if (!task) {
12421343
return;

0 commit comments

Comments
 (0)