Skip to content

Commit c0d8294

Browse files
claudeclaude
committed
bench: stabilise CodSpeed measurements
The previous CodSpeed run on #204 showed ±10-16% drift on tasks that weren't touched by the buffers() work (e.g. cached-source: size() (cached), cached-source: source() (cold), concat-source: getChildren()). Root cause is module-level shared state (warmed, warmedConcat, sink, *SourceLike) that grows whenever a new task is added to a case file. That state perturbs V8's hidden-class cache and GC heap layout for every task in the file, so adding a new task shifts pre-existing tasks' measurements. Two related fixes: 1. Teach the CodSpeed bridge about tinybench's beforeAll/beforeEach/ afterEach/afterAll hooks. The walltime path already honored them via tinybench itself, but the simulation path was just calling task.fn() raw, so any hook-based fixture setup was ignored. The bridge now runs beforeAll before warmup, beforeEach around every iteration (warmup and instrumented), afterEach after each, and afterAll after the instrumented pass. global.gc() still runs right before the instrumented call, after beforeEach. 2. Move heavy fixtures out of module scope into per-task beforeAll closures. Each case file's warmed/warmedConcat/warmLayeredChunk/ sourceLike/richSourceLike/sink now lives inside register() and is assigned in beforeAll and nulled in afterAll, so the set of tasks in a file only retains memory for the task currently running. Adding a future task to any of these files should no longer shift the pre-existing tasks' measurements. Also bumped warmupIterations from 2 to 10 in run.mjs so V8 hidden-class caches and the GC heap settle before the measured iteration. Side-effect visible in wall-clock numbers: new ConcatSource buffer/ buffers tasks now measure just the method call (construction moved to beforeAll), so the ratio grew from ~3x to ~10x on the flat 10-raw case and from ~2.4x to ~12x on nested 4x10. That's the honest comparison — the previous number was diluted by per-task fixture construction.
1 parent bb67759 commit c0d8294

6 files changed

Lines changed: 422 additions & 157 deletions

File tree

‎benchmark/cases/cached-source/index.bench.mjs‎

Lines changed: 138 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -4,27 +4,26 @@
44
* Two axes matter for CachedSource: cold vs warm (first call vs repeat call
55
* on the same instance), and the cache-data round-trip that lets webpack
66
* store cached state to disk via getCachedData() / re-hydrate.
7+
*
8+
* Fixture lifetime policy:
9+
* - Heavy fixtures (`warmed`, `warmedConcat`, `sink`) live inside
10+
* beforeAll/afterAll hooks so they are GC'd between tasks. This keeps
11+
* the V8 heap layout independent of how many tasks the file exports —
12+
* adding a new task does not perturb pre-existing tasks' measurements.
13+
* - Light fixtures (fixtureCode, fixtureMap) stay at module scope; they
14+
* are immutable and cheap to retain.
715
*/
816

917
import { createHash } from "crypto";
1018
import sources from "../../../lib/index.js";
1119
import { fixtureCode, fixtureMap, noop } from "../../fixtures.mjs";
1220

13-
/**
14-
* Pre-sized sink used by allocation-heavy tasks to retain the constructed
15-
* instances past the loop so V8 cannot dead-code-eliminate them and so
16-
* memory pressure is predictable between samples. Overwriting existing
17-
* slots (rather than push/length reset) keeps the sink's hidden class
18-
* stable and avoids resize allocations during measurement.
19-
*/
2021
const CONSTRUCT_BATCH = 100;
21-
const sink = Array.from({ length: CONSTRUCT_BATCH });
2222

2323
/**
24-
* A CachedSource with all the common caches already populated. Reused
25-
* across tasks that explicitly measure the warm path.
24+
* @returns {CachedSource} warmed CachedSource with all common caches populated
2625
*/
27-
const warmed = (() => {
26+
function makeWarmed() {
2827
const cached = new sources.CachedSource(
2928
new sources.SourceMapSource(fixtureCode, "fixture.js", fixtureMap),
3029
);
@@ -34,47 +33,87 @@ const warmed = (() => {
3433
cached.buffer();
3534
cached.size();
3635
return cached;
37-
})();
36+
}
3837

3938
/**
40-
* A CachedSource wrapping a ConcatSource of 10 RawSources with buffers()
41-
* already populated. Used to measure warm buffers()/buffer() delegation.
39+
* @returns {CachedSource} CachedSource wrapping a ConcatSource of 10 RawSources,
40+
* with buffers() already populated
4241
*/
43-
const warmedConcat = (() => {
42+
function makeWarmedConcat() {
4443
const parts = [];
4544
for (let i = 0; i < 10; i++) parts.push(new sources.RawSource(fixtureCode));
4645
const cached = new sources.CachedSource(new sources.ConcatSource(...parts));
4746
cached.buffers();
4847
return cached;
49-
})();
48+
}
5049

5150
/**
5251
* @param {import("tinybench").Bench} bench bench
5352
*/
5453
export default function register(bench) {
55-
bench.add("cached-source: new CachedSource()", () => {
56-
for (let i = 0; i < CONSTRUCT_BATCH; i++) {
57-
sink[i] = new sources.CachedSource(new sources.RawSource(fixtureCode));
58-
}
59-
});
54+
/** @type {unknown[] | undefined} */
55+
let sink;
56+
bench.add(
57+
"cached-source: new CachedSource()",
58+
() => {
59+
const arr = /** @type {unknown[]} */ (sink);
60+
for (let i = 0; i < CONSTRUCT_BATCH; i++) {
61+
arr[i] = new sources.CachedSource(new sources.RawSource(fixtureCode));
62+
}
63+
},
64+
{
65+
beforeAll() {
66+
sink = Array.from({ length: CONSTRUCT_BATCH });
67+
},
68+
afterAll() {
69+
sink = undefined;
70+
},
71+
},
72+
);
6073

6174
bench.add("cached-source: source() (cold)", () => {
6275
for (let i = 0; i < 50; i++) {
6376
new sources.CachedSource(new sources.RawSource(fixtureCode)).source();
6477
}
6578
});
6679

67-
bench.add("cached-source: source() (cached)", () => {
68-
for (let i = 0; i < 500; i++) warmed.source();
69-
});
80+
/** @type {CachedSource | undefined} */
81+
let warmed;
82+
const warmedHooks = {
83+
beforeAll() {
84+
warmed = makeWarmed();
85+
},
86+
afterAll() {
87+
warmed = undefined;
88+
},
89+
};
7090

71-
bench.add("cached-source: buffer() (cached)", () => {
72-
for (let i = 0; i < 500; i++) warmed.buffer();
73-
});
91+
bench.add(
92+
"cached-source: source() (cached)",
93+
() => {
94+
const cs = /** @type {CachedSource} */ (warmed);
95+
for (let i = 0; i < 500; i++) cs.source();
96+
},
97+
warmedHooks,
98+
);
7499

75-
bench.add("cached-source: buffers() (cached)", () => {
76-
for (let i = 0; i < 500; i++) warmed.buffers();
77-
});
100+
bench.add(
101+
"cached-source: buffer() (cached)",
102+
() => {
103+
const cs = /** @type {CachedSource} */ (warmed);
104+
for (let i = 0; i < 500; i++) cs.buffer();
105+
},
106+
warmedHooks,
107+
);
108+
109+
bench.add(
110+
"cached-source: buffers() (cached)",
111+
() => {
112+
const cs = /** @type {CachedSource} */ (warmed);
113+
for (let i = 0; i < 500; i++) cs.buffers();
114+
},
115+
warmedHooks,
116+
);
78117

79118
bench.add("cached-source: buffer() (cold, wraps ConcatSource x10)", () => {
80119
for (let i = 0; i < 10; i++) {
@@ -96,17 +135,43 @@ export default function register(bench) {
96135
}
97136
});
98137

99-
bench.add("cached-source: buffer() (warm, wraps ConcatSource x10)", () => {
100-
for (let i = 0; i < 500; i++) warmedConcat.buffer();
101-
});
138+
/** @type {CachedSource | undefined} */
139+
let warmedConcat;
140+
const warmedConcatHooks = {
141+
beforeAll() {
142+
warmedConcat = makeWarmedConcat();
143+
},
144+
afterAll() {
145+
warmedConcat = undefined;
146+
},
147+
};
102148

103-
bench.add("cached-source: buffers() (warm, wraps ConcatSource x10)", () => {
104-
for (let i = 0; i < 500; i++) warmedConcat.buffers();
105-
});
149+
bench.add(
150+
"cached-source: buffer() (warm, wraps ConcatSource x10)",
151+
() => {
152+
const cs = /** @type {CachedSource} */ (warmedConcat);
153+
for (let i = 0; i < 500; i++) cs.buffer();
154+
},
155+
warmedConcatHooks,
156+
);
106157

107-
bench.add("cached-source: size() (cached)", () => {
108-
for (let i = 0; i < 500; i++) warmed.size();
109-
});
158+
bench.add(
159+
"cached-source: buffers() (warm, wraps ConcatSource x10)",
160+
() => {
161+
const cs = /** @type {CachedSource} */ (warmedConcat);
162+
for (let i = 0; i < 500; i++) cs.buffers();
163+
},
164+
warmedConcatHooks,
165+
);
166+
167+
bench.add(
168+
"cached-source: size() (cached)",
169+
() => {
170+
const cs = /** @type {CachedSource} */ (warmed);
171+
for (let i = 0; i < 500; i++) cs.size();
172+
},
173+
warmedHooks,
174+
);
110175

111176
bench.add("cached-source: map() (cold SourceMapSource)", () => {
112177
for (let i = 0; i < 10; i++) {
@@ -116,9 +181,14 @@ export default function register(bench) {
116181
}
117182
});
118183

119-
bench.add("cached-source: map() (cached)", () => {
120-
for (let i = 0; i < 500; i++) warmed.map({});
121-
});
184+
bench.add(
185+
"cached-source: map() (cached)",
186+
() => {
187+
const cs = /** @type {CachedSource} */ (warmed);
188+
for (let i = 0; i < 500; i++) cs.map({});
189+
},
190+
warmedHooks,
191+
);
122192

123193
bench.add("cached-source: sourceAndMap() (cold)", () => {
124194
for (let i = 0; i < 10; i++) {
@@ -128,9 +198,14 @@ export default function register(bench) {
128198
}
129199
});
130200

131-
bench.add("cached-source: sourceAndMap() (cached)", () => {
132-
for (let i = 0; i < 500; i++) warmed.sourceAndMap({});
133-
});
201+
bench.add(
202+
"cached-source: sourceAndMap() (cached)",
203+
() => {
204+
const cs = /** @type {CachedSource} */ (warmed);
205+
for (let i = 0; i < 500; i++) cs.sourceAndMap({});
206+
},
207+
warmedHooks,
208+
);
134209

135210
bench.add("cached-source: streamChunks() (cold)", () => {
136211
for (let i = 0; i < 5; i++) {
@@ -140,11 +215,16 @@ export default function register(bench) {
140215
}
141216
});
142217

143-
bench.add("cached-source: streamChunks() (warm)", () => {
144-
for (let i = 0; i < 5; i++) {
145-
warmed.streamChunks({}, noop, noop, noop);
146-
}
147-
});
218+
bench.add(
219+
"cached-source: streamChunks() (warm)",
220+
() => {
221+
const cs = /** @type {CachedSource} */ (warmed);
222+
for (let i = 0; i < 5; i++) {
223+
cs.streamChunks({}, noop, noop, noop);
224+
}
225+
},
226+
warmedHooks,
227+
);
148228

149229
bench.add("cached-source: originalLazy()", () => {
150230
const lazy = new sources.CachedSource(
@@ -168,7 +248,12 @@ export default function register(bench) {
168248
}
169249
});
170250

171-
bench.add("cached-source: updateHash() (warm)", () => {
172-
for (let i = 0; i < 50; i++) warmed.updateHash(createHash("sha256"));
173-
});
251+
bench.add(
252+
"cached-source: updateHash() (warm)",
253+
() => {
254+
const cs = /** @type {CachedSource} */ (warmed);
255+
for (let i = 0; i < 50; i++) cs.updateHash(createHash("sha256"));
256+
},
257+
warmedHooks,
258+
);
174259
}

0 commit comments

Comments
 (0)