Skip to content

Commit 9788e55

Browse files
alexander-akaitalexander-akait
authored
feat: add buffers() method returning Buffer[] (#204)
1 parent 58420d1 commit 9788e55

20 files changed

Lines changed: 740 additions & 99 deletions

‎.changeset/buffers-method.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"webpack-sources": minor
3+
---
4+
5+
Add `Source.prototype.buffers()` that returns the source as `Buffer[]`. `ConcatSource`, `CachedSource`, and `CompatSource` implement it without allocating an intermediate concatenated buffer, allowing consumers that can write multiple buffers at once (e.g. via `writev`) to avoid the overhead of `Buffer.concat` in deeply nested sources.

‎README.md‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,21 @@ Source.prototype.buffer() -> Buffer
2828

2929
Returns the represented source code as Buffer. Strings are converted to utf-8.
3030

31+
#### `buffers`
32+
33+
<!-- eslint-skip -->
34+
```typescript
35+
Source.prototype.buffers() -> Buffer[]
36+
```
37+
38+
Returns the represented source code as an array of Buffers. This avoids the
39+
intermediate `Buffer.concat` allocation performed by `buffer()` when the source
40+
is composed of multiple children (for example `ConcatSource`). Consumers that
41+
can accept an array of buffers (e.g. writing via `fs.createWriteStream` or
42+
`writev`) should prefer this method for better performance.
43+
44+
The default implementation returns `[this.buffer()]`.
45+
3146
#### `size`
3247

3348
<!-- eslint-skip -->

‎benchmark/README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,7 @@ stable than per-call timing for sub-microsecond work.
112112

113113
| Case | What it measures |
114114
| --------------------------------- | ----------------------------------------------------------------------------- |
115-
| `realistic-source-map-pipeline` | OriginalSource -> ReplaceSource -> ConcatSource -> CachedSource (cold + warm) |
115+
| `realistic-source-map-pipeline` | OriginalSource -> ReplaceSource -> ConcatSource -> CachedSource (cold + warm); also `buffer()` vs `buffers()` over the `CachedSource -> ConcatSource -> CachedSource -> ConcatSource` layering from issue #157 |
116116

117117
## Adding a new case
118118

‎benchmark/cases/cached-source/index.bench.mjs‎

Lines changed: 167 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -4,27 +4,26 @@
44
* Two axes matter for CachedSource: cold vs warm (first call vs repeat call
55
* on the same instance), and the cache-data round-trip that lets webpack
66
* store cached state to disk via getCachedData() / re-hydrate.
7+
*
8+
* Fixture lifetime policy:
9+
* - Heavy fixtures (`warmed`, `warmedConcat`, `sink`) live inside
10+
* beforeAll/afterAll hooks so they are GC'd between tasks. This keeps
11+
* the V8 heap layout independent of how many tasks the file exports —
12+
* adding a new task does not perturb pre-existing tasks' measurements.
13+
* - Light fixtures (fixtureCode, fixtureMap) stay at module scope; they
14+
* are immutable and cheap to retain.
715
*/
816

917
import { createHash } from "crypto";
1018
import sources from "../../../lib/index.js";
1119
import { fixtureCode, fixtureMap, noop } from "../../fixtures.mjs";
1220

13-
/**
14-
* Pre-sized sink used by allocation-heavy tasks to retain the constructed
15-
* instances past the loop so V8 cannot dead-code-eliminate them and so
16-
* memory pressure is predictable between samples. Overwriting existing
17-
* slots (rather than push/length reset) keeps the sink's hidden class
18-
* stable and avoids resize allocations during measurement.
19-
*/
2021
const CONSTRUCT_BATCH = 100;
21-
const sink = Array.from({ length: CONSTRUCT_BATCH });
2222

2323
/**
24-
* A CachedSource with all the common caches already populated. Reused
25-
* across tasks that explicitly measure the warm path.
24+
* @returns {CachedSource} warmed CachedSource with all common caches populated
2625
*/
27-
const warmed = (() => {
26+
function makeWarmed() {
2827
const cached = new sources.CachedSource(
2928
new sources.SourceMapSource(fixtureCode, "fixture.js", fixtureMap),
3029
);
@@ -34,36 +33,146 @@ const warmed = (() => {
3433
cached.buffer();
3534
cached.size();
3635
return cached;
37-
})();
36+
}
37+
38+
/**
39+
* @returns {CachedSource} CachedSource wrapping a ConcatSource of 10 RawSources,
40+
* with buffers() already populated
41+
*/
42+
function makeWarmedConcat() {
43+
const parts = [];
44+
for (let i = 0; i < 10; i++) parts.push(new sources.RawSource(fixtureCode));
45+
const cached = new sources.CachedSource(new sources.ConcatSource(...parts));
46+
cached.buffers();
47+
return cached;
48+
}
3849

3950
/**
4051
* @param {import("tinybench").Bench} bench bench
4152
*/
4253
export default function register(bench) {
43-
bench.add("cached-source: new CachedSource()", () => {
44-
for (let i = 0; i < CONSTRUCT_BATCH; i++) {
45-
sink[i] = new sources.CachedSource(new sources.RawSource(fixtureCode));
46-
}
47-
});
54+
/** @type {unknown[] | undefined} */
55+
let sink;
56+
bench.add(
57+
"cached-source: new CachedSource()",
58+
() => {
59+
const arr = /** @type {unknown[]} */ (sink);
60+
for (let i = 0; i < CONSTRUCT_BATCH; i++) {
61+
arr[i] = new sources.CachedSource(new sources.RawSource(fixtureCode));
62+
}
63+
},
64+
{
65+
beforeAll() {
66+
sink = Array.from({ length: CONSTRUCT_BATCH });
67+
},
68+
afterAll() {
69+
sink = undefined;
70+
},
71+
},
72+
);
4873

4974
bench.add("cached-source: source() (cold)", () => {
5075
for (let i = 0; i < 50; i++) {
5176
new sources.CachedSource(new sources.RawSource(fixtureCode)).source();
5277
}
5378
});
5479

55-
bench.add("cached-source: source() (cached)", () => {
56-
for (let i = 0; i < 500; i++) warmed.source();
57-
});
80+
/** @type {CachedSource | undefined} */
81+
let warmed;
82+
const warmedHooks = {
83+
beforeAll() {
84+
warmed = makeWarmed();
85+
},
86+
afterAll() {
87+
warmed = undefined;
88+
},
89+
};
90+
91+
bench.add(
92+
"cached-source: source() (cached)",
93+
() => {
94+
const cs = /** @type {CachedSource} */ (warmed);
95+
for (let i = 0; i < 500; i++) cs.source();
96+
},
97+
warmedHooks,
98+
);
5899

59-
bench.add("cached-source: buffer() (cached)", () => {
60-
for (let i = 0; i < 500; i++) warmed.buffer();
100+
bench.add(
101+
"cached-source: buffer() (cached)",
102+
() => {
103+
const cs = /** @type {CachedSource} */ (warmed);
104+
for (let i = 0; i < 500; i++) cs.buffer();
105+
},
106+
warmedHooks,
107+
);
108+
109+
bench.add(
110+
"cached-source: buffers() (cached)",
111+
() => {
112+
const cs = /** @type {CachedSource} */ (warmed);
113+
for (let i = 0; i < 500; i++) cs.buffers();
114+
},
115+
warmedHooks,
116+
);
117+
118+
bench.add("cached-source: buffer() (cold, wraps ConcatSource x10)", () => {
119+
for (let i = 0; i < 10; i++) {
120+
const parts = [];
121+
for (let j = 0; j < 10; j++) {
122+
parts.push(new sources.RawSource(fixtureCode));
123+
}
124+
new sources.CachedSource(new sources.ConcatSource(...parts)).buffer();
125+
}
61126
});
62127

63-
bench.add("cached-source: size() (cached)", () => {
64-
for (let i = 0; i < 500; i++) warmed.size();
128+
bench.add("cached-source: buffers() (cold, wraps ConcatSource x10)", () => {
129+
for (let i = 0; i < 10; i++) {
130+
const parts = [];
131+
for (let j = 0; j < 10; j++) {
132+
parts.push(new sources.RawSource(fixtureCode));
133+
}
134+
new sources.CachedSource(new sources.ConcatSource(...parts)).buffers();
135+
}
65136
});
66137

138+
/** @type {CachedSource | undefined} */
139+
let warmedConcat;
140+
const warmedConcatHooks = {
141+
beforeAll() {
142+
warmedConcat = makeWarmedConcat();
143+
},
144+
afterAll() {
145+
warmedConcat = undefined;
146+
},
147+
};
148+
149+
bench.add(
150+
"cached-source: buffer() (warm, wraps ConcatSource x10)",
151+
() => {
152+
const cs = /** @type {CachedSource} */ (warmedConcat);
153+
for (let i = 0; i < 500; i++) cs.buffer();
154+
},
155+
warmedConcatHooks,
156+
);
157+
158+
bench.add(
159+
"cached-source: buffers() (warm, wraps ConcatSource x10)",
160+
() => {
161+
const cs = /** @type {CachedSource} */ (warmedConcat);
162+
for (let i = 0; i < 500; i++) cs.buffers();
163+
},
164+
warmedConcatHooks,
165+
);
166+
167+
bench.add(
168+
"cached-source: size() (cached)",
169+
() => {
170+
const cs = /** @type {CachedSource} */ (warmed);
171+
for (let i = 0; i < 500; i++) cs.size();
172+
},
173+
warmedHooks,
174+
);
175+
67176
bench.add("cached-source: map() (cold SourceMapSource)", () => {
68177
for (let i = 0; i < 10; i++) {
69178
new sources.CachedSource(
@@ -72,9 +181,14 @@ export default function register(bench) {
72181
}
73182
});
74183

75-
bench.add("cached-source: map() (cached)", () => {
76-
for (let i = 0; i < 500; i++) warmed.map({});
77-
});
184+
bench.add(
185+
"cached-source: map() (cached)",
186+
() => {
187+
const cs = /** @type {CachedSource} */ (warmed);
188+
for (let i = 0; i < 500; i++) cs.map({});
189+
},
190+
warmedHooks,
191+
);
78192

79193
bench.add("cached-source: sourceAndMap() (cold)", () => {
80194
for (let i = 0; i < 10; i++) {
@@ -84,9 +198,14 @@ export default function register(bench) {
84198
}
85199
});
86200

87-
bench.add("cached-source: sourceAndMap() (cached)", () => {
88-
for (let i = 0; i < 500; i++) warmed.sourceAndMap({});
89-
});
201+
bench.add(
202+
"cached-source: sourceAndMap() (cached)",
203+
() => {
204+
const cs = /** @type {CachedSource} */ (warmed);
205+
for (let i = 0; i < 500; i++) cs.sourceAndMap({});
206+
},
207+
warmedHooks,
208+
);
90209

91210
bench.add("cached-source: streamChunks() (cold)", () => {
92211
for (let i = 0; i < 5; i++) {
@@ -96,11 +215,16 @@ export default function register(bench) {
96215
}
97216
});
98217

99-
bench.add("cached-source: streamChunks() (warm)", () => {
100-
for (let i = 0; i < 5; i++) {
101-
warmed.streamChunks({}, noop, noop, noop);
102-
}
103-
});
218+
bench.add(
219+
"cached-source: streamChunks() (warm)",
220+
() => {
221+
const cs = /** @type {CachedSource} */ (warmed);
222+
for (let i = 0; i < 5; i++) {
223+
cs.streamChunks({}, noop, noop, noop);
224+
}
225+
},
226+
warmedHooks,
227+
);
104228

105229
bench.add("cached-source: originalLazy()", () => {
106230
const lazy = new sources.CachedSource(
@@ -124,7 +248,12 @@ export default function register(bench) {
124248
}
125249
});
126250

127-
bench.add("cached-source: updateHash() (warm)", () => {
128-
for (let i = 0; i < 50; i++) warmed.updateHash(createHash("sha256"));
129-
});
251+
bench.add(
252+
"cached-source: updateHash() (warm)",
253+
() => {
254+
const cs = /** @type {CachedSource} */ (warmed);
255+
for (let i = 0; i < 50; i++) cs.updateHash(createHash("sha256"));
256+
},
257+
warmedHooks,
258+
);
130259
}

0 commit comments

Comments
 (0)