Skip to content

Commit 0e3958f

Browse files
perf: release a module source's redundant buffer after hashing
Hashing a module's source reads its bytes, so a string-backed source comes out of the build hash holding its content twice: once as the string the parser read and once as the buffer the hash consumed. On a 1033-module build that is 2.71 MB of buffers retained for nothing. Release the copy once the hash is taken, where the source carries no map — a map-carrying source keeps the buffer and drops the string, which the generator then rebuilds, so it is left alone.
1 parent 951d0c6 commit 0e3958f

3 files changed

Lines changed: 131 additions & 0 deletions

File tree

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"webpack": patch
3+
---
4+
5+
Release a module source's redundant buffer once its build hash is taken.

‎lib/module/NormalModule.js‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,10 @@ const getModuleBuildError = memoize(() =>
8383
require("../errors/ModuleBuildError")
8484
);
8585

86+
// Drops the redundant string/buffer copy of a source, keeping its map: a
87+
// parsed map is far more expensive to rebuild than a `toString`.
88+
const CLEAR_SOURCE_COPIES = { maps: false, source: true };
89+
8690
/** @import { ResolveContext, ResolveRequest, Resolver } from "enhanced-resolve" */
8791
/** @import { Source, RawSourceMap } from "webpack-sources" */
8892
/**
@@ -1657,6 +1661,18 @@ class NormalModule extends Module {
16571661
if (this._source) {
16581662
hash.update("source");
16591663
this._source.updateHash(hash);
1664+
// WHY: hashing reads the bytes, so the source now holds its content
1665+
// as a string and as a buffer. Dropping one is only a win where the
1666+
// buffer is the copy: a map-carrying source keeps the buffer and
1667+
// drops the string, which the generator then rebuilds.
1668+
// Feature-detected, since `clearCache` landed in webpack-sources 3.5.
1669+
if (
1670+
!this.useSourceMap &&
1671+
!this.useSimpleSourceMap &&
1672+
typeof this._source.clearCache === "function"
1673+
) {
1674+
this._source.clearCache(CLEAR_SOURCE_COPIES);
1675+
}
16601676
}
16611677
hash.update("meta");
16621678
hash.update(JSON.stringify(this.buildMeta));
Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
"use strict";
2+
3+
require("../helpers/warmup-webpack");
4+
5+
const fs = require("fs");
6+
const path = require("path");
7+
const { Volume, createFsFromVolume } = require("memfs");
8+
9+
/** @typedef {import("../../").NormalModule} NormalModule */
10+
11+
const CONTEXT = path.join(__dirname, "..", "fixtures");
12+
13+
/** @typedef {{ name: string, string: boolean, buffer: boolean, text: string }} Reading */
14+
15+
/**
16+
* Which cached forms of its content a module's source holds, plus the text it
17+
* serves. Both form fields are webpack-sources internals, read never written.
18+
* @param {NormalModule} module the built module
19+
* @returns {Reading} what the source holds right now
20+
*/
21+
const read = (module) => {
22+
const source =
23+
/** @type {import("webpack-sources").Source & { _value?: unknown, _valueAsString?: unknown, _valueAsBuffer?: unknown }} */
24+
(module.originalSource());
25+
return {
26+
name: path.basename(module.identifier()),
27+
string:
28+
typeof source._valueAsString === "string" ||
29+
typeof source._value === "string",
30+
buffer:
31+
Buffer.isBuffer(source._valueAsBuffer) || Buffer.isBuffer(source._value),
32+
text: source.source().toString()
33+
};
34+
};
35+
36+
/**
37+
* Runs one compilation and reports what each module's source held the moment
38+
* the graph was finished — the point the release is about, since rendering
39+
* reads a source's bytes again afterwards.
40+
* @param {import("../../").Configuration} options webpack options
41+
* @returns {Promise<Map<string, Reading>>} one reading per module, by resource
42+
*/
43+
const readingsAtFinishModules = (options) =>
44+
new Promise((resolve, reject) => {
45+
const webpack = require("../..");
46+
47+
/** @type {Map<string, Reading>} */
48+
const readings = new Map();
49+
const compiler = webpack({
50+
mode: "production",
51+
context: CONTEXT,
52+
entry: "./main1.js",
53+
optimization: { minimize: false },
54+
...options
55+
});
56+
compiler.outputFileSystem =
57+
/** @type {import("../../").OutputFileSystem} */
58+
(/** @type {unknown} */ (createFsFromVolume(new Volume())));
59+
compiler.hooks.compilation.tap("ModuleSourceRelease", (compilation) => {
60+
compilation.hooks.finishModules.tap("ModuleSourceRelease", (modules) => {
61+
for (const module of modules) {
62+
const normalModule = /** @type {NormalModule} */ (module);
63+
if (normalModule.originalSource() === null) continue;
64+
readings.set(normalModule.resource, read(normalModule));
65+
}
66+
});
67+
});
68+
compiler.run((err, stats) => {
69+
if (err) return reject(err);
70+
if (!stats) return reject(new Error("no stats"));
71+
if (stats.hasErrors()) {
72+
return reject(new Error(stats.toString({ errors: true })));
73+
}
74+
compiler.close((closeError) =>
75+
closeError ? reject(closeError) : resolve(readings)
76+
);
77+
});
78+
});
79+
80+
describe("module source release", () => {
81+
it("leaves one cached form per source when no source map is built", async () => {
82+
const readings = await readingsAtFinishModules({ devtool: false });
83+
expect(readings.size).toBeGreaterThan(0);
84+
for (const [, reading] of readings) {
85+
expect({
86+
name: reading.name,
87+
string: reading.string,
88+
buffer: reading.buffer
89+
}).toEqual({ name: reading.name, string: true, buffer: false });
90+
}
91+
});
92+
93+
it("still serves each source's text after releasing the other form", async () => {
94+
const readings = await readingsAtFinishModules({ devtool: false });
95+
for (const [resource, reading] of readings) {
96+
expect(reading.text).toBe(fs.readFileSync(resource, "utf8"));
97+
}
98+
});
99+
100+
it("keeps the string a map-carrying source's generator reads back", async () => {
101+
const readings = await readingsAtFinishModules({ devtool: "source-map" });
102+
expect(readings.size).toBeGreaterThan(0);
103+
for (const [, reading] of readings) {
104+
expect({ name: reading.name, string: reading.string }).toEqual({
105+
name: reading.name,
106+
string: true
107+
});
108+
}
109+
});
110+
});

0 commit comments

Comments
 (0)