@@ -8,11 +8,17 @@ import { fileURLToPath, pathToFileURL } from "url";
88import vm from "vm" ;
99import {
1010 InstrumentHooks ,
11+ MARKER_TYPE_BENCHMARK_END ,
12+ MARKER_TYPE_BENCHMARK_START ,
13+ calculateQuantiles ,
1114 getCodspeedRunnerMode ,
1215 getGitDir ,
1316 mongoMeasurement ,
17+ msToNs ,
18+ msToS ,
1419 setupCore ,
15- teardownCore
20+ teardownCore ,
21+ writeWalltimeResults
1622} from "@codspeed/core" ;
1723import { Bench , hrtimeNow } from "tinybench" ;
1824
@@ -25,6 +31,7 @@ import { Bench, hrtimeNow } from "tinybench";
2531 * } from "tinybench"
2632 */
2733/** @typedef {TinybenchTask & { collectBy?: string } } Task */
34+ /** @import { Benchmark } from "@codspeed/core" */
2835
2936/** @import Webpack, { Configuration, Stats, Watching } from "../../.." */
3037/**
@@ -262,6 +269,23 @@ function getTaskUri(bench, taskName, callingFile) {
262269 return uriMap ?. get ( taskName ) ?. uri || `${ callingFile } ::${ taskName } ` ;
263270}
264271
272+ /**
273+ * @param {Fn } fn function
274+ * @param {boolean } isAsync true if async function, otherwise false
275+ * @returns {() => void } function wrapped in codspeed root frame
276+ */
277+ const wrapFunctionWithFrame = ( fn , isAsync ) => {
278+ if ( isAsync ) {
279+ return async function __codspeed_root_frame__ ( ) {
280+ await fn ( ) ;
281+ } ;
282+ }
283+
284+ return function __codspeed_root_frame__ ( ) {
285+ fn ( ) ;
286+ } ;
287+ } ;
288+
265289/**
266290 * @param {Bench } bench bench
267291 * @returns {Promise<Bench> } modifier bench
@@ -288,6 +312,15 @@ const withCodSpeed = async (bench) => {
288312 if ( releaseClosuresAfterMeasure ) {
289313 return rawAdd . bind ( bench ) ( name , NO_OP_FN , { } ) ;
290314 }
315+ if ( codspeedRunnerMode === "walltime" ) {
316+ // Walltime samples outside this frame are not attributed to the bench.
317+ const isAsync = fn . constructor . name === "AsyncFunction" ;
318+ return rawAdd . bind ( bench ) (
319+ name ,
320+ wrapFunctionWithFrame ( fn , isAsync ) ,
321+ options
322+ ) ;
323+ }
291324 return rawAdd . bind ( bench ) ( name , fn , options ) ;
292325 } ;
293326 const callingFile = getCallingFile ( ) ;
@@ -311,23 +344,6 @@ const withCodSpeed = async (bench) => {
311344 return bench . tasks ;
312345 } ;
313346
314- /**
315- * @param {Fn } fn function
316- * @param {boolean } isAsync true if async function, otherwise false
317- * @returns {() => void } function wrapped in codspeed root frame
318- */
319- const wrapFunctionWithFrame = ( fn , isAsync ) => {
320- if ( isAsync ) {
321- return async function __codspeed_root_frame__ ( ) {
322- await fn ( ) ;
323- } ;
324- }
325-
326- return function __codspeed_root_frame__ ( ) {
327- fn ( ) ;
328- } ;
329- } ;
330-
331347 /**
332348 * @param {string } uri URI
333349 * @param {string } status status
@@ -653,12 +669,95 @@ const withCodSpeed = async (bench) => {
653669 return finalizeSyncRun ( ) ;
654670 } ;
655671 } else if ( codspeedRunnerMode === "walltime" ) {
656- // We don't need it
672+ /** @type {Benchmark[] } */
673+ const codspeedBenchmarks = [ ] ;
674+
675+ bench . run = async ( ) => {
676+ setupCore ( ) ;
677+ console . log (
678+ `[CodSpeed] running with @codspeed/tinybench (walltime mode, ${ bench . warmupIterations } warmup iterations)`
679+ ) ;
680+
681+ for ( const task of bench . tasks ) {
682+ const uri = getTaskUri ( bench , task . name , callingFile ) ;
683+
684+ await task . warmup ( ) ;
685+ InstrumentHooks . startBenchmark ( ) ;
686+ const runStart = InstrumentHooks . currentTimestamp ( ) ;
687+ await task . run ( ) ;
688+ const runEnd = InstrumentHooks . currentTimestamp ( ) ;
689+ InstrumentHooks . stopBenchmark ( ) ;
690+ InstrumentHooks . setExecutedBenchmark ( process . pid , uri ) ;
691+ InstrumentHooks . addMarker (
692+ process . pid ,
693+ MARKER_TYPE_BENCHMARK_START ,
694+ runStart
695+ ) ;
696+ InstrumentHooks . addMarker (
697+ process . pid ,
698+ MARKER_TYPE_BENCHMARK_END ,
699+ runEnd
700+ ) ;
701+
702+ codspeedBenchmarks . push ( toWalltimeBenchmark ( bench , task , uri ) ) ;
703+ console . log ( `[CodSpeed] Collected walltime data for ${ uri } ` ) ;
704+ }
705+
706+ writeWalltimeResults ( codspeedBenchmarks , true ) ;
707+ teardownCore ( ) ;
708+ console . log ( `[CodSpeed] Done running ${ bench . tasks . length } benches.` ) ;
709+ return bench . tasks ;
710+ } ;
657711 }
658712
659713 return bench ;
660714} ;
661715
716+ /**
717+ * Convert a finished tinybench task into CodSpeed's walltime result entry.
718+ * Tinybench runs one round, reported as one round of one iteration per sample.
719+ * @param {Bench } bench bench
720+ * @param {TinybenchTask } task finished task, run with `retainSamples`
721+ * @param {string } uri task URI
722+ * @returns {Benchmark } walltime result entry
723+ */
724+ function toWalltimeBenchmark ( bench , task , uri ) {
725+ const result = task . result ;
726+
727+ if ( ! result || result . state !== "completed" ) {
728+ throw new Error ( `No walltime result for ${ uri } ` ) ;
729+ }
730+
731+ const { min, max, mean, sd, samples } = result . latency ;
732+ const sortedTimesNs = /** @type {number[] } */ ( samples )
733+ . map ( msToNs )
734+ . sort ( ( a , b ) => a - b ) ;
735+ const meanNs = msToNs ( mean ) ;
736+ const stdevNs = msToNs ( sd ) ;
737+
738+ return {
739+ name : task . name ,
740+ uri,
741+ config : {
742+ max_rounds : bench . iterations ,
743+ max_time_ns : msToNs ( bench . time ) ,
744+ min_round_time_ns : null ,
745+ warmup_time_ns : msToNs ( bench . warmupTime )
746+ } ,
747+ stats : {
748+ min_ns : msToNs ( min ) ,
749+ max_ns : msToNs ( max ) ,
750+ mean_ns : meanNs ,
751+ stdev_ns : stdevNs ,
752+ ...calculateQuantiles ( { sortedTimesNs, stdevNs, meanNs } ) ,
753+ total_time : msToS ( result . totalTime ) ,
754+ iter_per_round : 1 ,
755+ rounds : sortedTimesNs . length ,
756+ warmup_iters : bench . warmupIterations
757+ }
758+ } ;
759+ }
760+
662761/**
663762 * @param {number } n number of runs
664763 * @returns {number } distribution
@@ -1237,6 +1336,8 @@ function createBenchInstance() {
12371336 warmup : true ,
12381337 warmupIterations : 2 ,
12391338 iterations : 8 ,
1339+ // Walltime reports quantiles, computed from the raw samples.
1340+ retainSamples : codspeedRunnerMode === "walltime" ,
12401341 setup ( task , mode ) {
12411342 if ( ! task ) {
12421343 return ;
0 commit comments