Skip to content

Commit 2d6688d

Browse files
alexander-akaitalexander-akait
committed
fix: release stale compilation data from nested children and idle watches
Compiler._cleanupLastCompilation walked compilation.children one level deep, so a module only reached below the first level kept its parser, generator and memoized ModuleGraph — the references NormalModule's own comments say must go, because they pin the whole stale compilation. _releaseUnusedCompilationData already recursed; this makes both agree. A watch run also goes idle rather than closing, so codeGenerationResults — rendered source per module and runtime, read only during seal and already dropped on close() — was held until the next build started. On a 1700-module fixture that is 6.3 MB of an 85.6 MB idle heap. Emitted assets are byte-identical before and after in development and production.
1 parent 55c9808 commit 2d6688d

4 files changed

Lines changed: 265 additions & 19 deletions

File tree

‎.changeset/018-stale-compilation-data.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"webpack": patch
3+
---
4+
5+
Release stale compilation data from nested children and idle watch builds.

‎lib/Compiler.js‎

Lines changed: 23 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -497,29 +497,33 @@ class Compiler {
497497
// e.g. move compilation specific info from Modules into ModuleGraph
498498
_cleanupLastCompilation() {
499499
if (this._lastCompilation !== undefined) {
500-
for (const childCompilation of this._lastCompilation.children) {
501-
for (const module of childCompilation.modules) {
502-
ChunkGraph.clearChunkGraphForModule(module);
503-
ModuleGraph.clearModuleGraphForModule(module);
504-
module.cleanupForCache();
505-
}
506-
for (const chunk of childCompilation.chunks) {
507-
ChunkGraph.clearChunkGraphForChunk(chunk);
508-
}
509-
}
510-
511-
for (const module of this._lastCompilation.modules) {
512-
ChunkGraph.clearChunkGraphForModule(module);
513-
ModuleGraph.clearModuleGraphForModule(module);
514-
module.cleanupForCache();
515-
}
516-
for (const chunk of this._lastCompilation.chunks) {
517-
ChunkGraph.clearChunkGraphForChunk(chunk);
518-
}
500+
this._cleanupCompilationForCache(this._lastCompilation);
519501
this._lastCompilation = undefined;
520502
}
521503
}
522504

505+
/**
506+
* Drops the graph references a finished compilation left on its modules and
507+
* chunks. Recurses, because child compilations nest: a module only reached
508+
* below the first level keeps its parser, generator and memoized
509+
* `ModuleGraph`, which pins the whole stale compilation.
510+
* @param {Compilation} compilation compilation to clean up
511+
* @returns {void}
512+
*/
513+
_cleanupCompilationForCache(compilation) {
514+
for (const childCompilation of compilation.children) {
515+
this._cleanupCompilationForCache(childCompilation);
516+
}
517+
for (const module of compilation.modules) {
518+
ChunkGraph.clearChunkGraphForModule(module);
519+
ModuleGraph.clearModuleGraphForModule(module);
520+
module.cleanupForCache();
521+
}
522+
for (const chunk of compilation.chunks) {
523+
ChunkGraph.clearChunkGraphForChunk(chunk);
524+
}
525+
}
526+
523527
// TODO webpack 6: solve this in a better way
524528
_cleanupLastNormalModuleFactory() {
525529
if (this._lastNormalModuleFactory !== undefined) {

‎lib/Watching.js‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -362,6 +362,11 @@ class Watching {
362362
});
363363
for (const cb of cbs) cb(null);
364364
this.compiler.hooks.afterDone.call(/** @type {Stats} */ (stats));
365+
// A watch run goes idle here rather than closing, so seal-only
366+
// data would be held until the next build starts.
367+
this.compiler._releaseUnusedCompilationData(
368+
/** @type {Compilation} */ (compilation)
369+
);
365370
}
366371
);
367372
});

‎test/CompilationRelease.test.js‎

Lines changed: 232 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,232 @@
1+
"use strict";
2+
3+
require("./helpers/warmup-webpack");
4+
5+
const path = require("path");
6+
const { Volume, createFsFromVolume } = require("memfs");
7+
8+
/** @typedef {import("../").Compilation} Compilation */
9+
/** @typedef {import("../").Compiler} Compiler */
10+
/** @typedef {import("../").Module} Module */
11+
/** @typedef {import("../").NormalModule} NormalModule */
12+
/** @typedef {import("../").Stats} Stats */
13+
/** @typedef {import("../").WebpackPluginInstance} WebpackPluginInstance */
14+
15+
const CONTEXT = path.join(__dirname, "fixtures");
16+
17+
// One fixture per nesting level, so every level owns a module instance no other
18+
// level shares — a shared instance is cleaned up by whichever level is reached.
19+
const LEVEL_ENTRIES = ["./a.js", "./b.js", "./main1.js", "./main2.js"];
20+
21+
/**
22+
* Builds a chain of child compilations `depth` levels deep, each created from
23+
* the compilation above it.
24+
* @param {number} depth how many nested child compilations to create
25+
* @returns {WebpackPluginInstance} plugin
26+
*/
27+
const nestedChildPlugin = (depth) => ({
28+
apply(compiler) {
29+
const webpack = require("..");
30+
31+
compiler.hooks.make.tapAsync(
32+
"NestedChildPlugin",
33+
(compilation, callback) => {
34+
/**
35+
* @param {Compilation} parent compilation to create the child under
36+
* @param {number} level current nesting level
37+
* @param {(err?: Error | null) => void} done completion callback
38+
* @returns {void}
39+
*/
40+
const build = (parent, level, done) => {
41+
const child = parent.createChildCompiler(`child-${level}`, {
42+
filename: `child-${level}.js`
43+
});
44+
child.hooks.make.tapAsync(
45+
"NestedChildPlugin",
46+
(childCompilation, cb) => {
47+
const dep = webpack.EntryPlugin.createDependency(
48+
LEVEL_ENTRIES[level + 1],
49+
{ name: `child-${level}` }
50+
);
51+
childCompilation.addEntry(
52+
CONTEXT,
53+
dep,
54+
`child-${level}`,
55+
(err) => {
56+
if (err) return cb(err);
57+
if (level + 1 < depth) {
58+
return build(childCompilation, level + 1, cb);
59+
}
60+
cb();
61+
}
62+
);
63+
}
64+
);
65+
child.runAsChild((err) => done(err));
66+
};
67+
build(compilation, 0, callback);
68+
}
69+
);
70+
}
71+
});
72+
73+
/**
74+
* The modules that still carry compilation-scoped helpers, by base name.
75+
* @param {Module[]} modules modules of one compilation
76+
* @returns {string[]} sorted base names of the modules not cleaned up
77+
*/
78+
const retainedModules = (modules) =>
79+
modules
80+
.filter((module) => {
81+
const normalModule = /** @type {NormalModule} */ (module);
82+
return (
83+
normalModule.parser !== undefined ||
84+
normalModule.generator !== undefined
85+
);
86+
})
87+
.map((module) => path.basename(module.identifier()))
88+
.sort();
89+
90+
/**
91+
* Flattens a compilation tree into one entry per compilation.
92+
* @param {Compilation} compilation root compilation
93+
* @param {number} level nesting level of `compilation`
94+
* @param {{ level: number, modules: Module[] }[]} out collected levels
95+
* @returns {{ level: number, modules: Module[] }[]} collected levels
96+
*/
97+
const collectLevels = (compilation, level, out) => {
98+
out.push({ level, modules: [...compilation.modules] });
99+
for (const child of compilation.children) {
100+
collectLevels(child, level + 1, out);
101+
}
102+
return out;
103+
};
104+
105+
/**
106+
* Runs one compilation and hands the compiler and stats to the caller before
107+
* closing it.
108+
* @param {import("../").Configuration} options webpack options
109+
* @returns {Promise<{ compiler: Compiler, stats: Stats, close: () => Promise<void> }>} the finished build
110+
*/
111+
const build = (options) =>
112+
new Promise((resolve, reject) => {
113+
const webpack = require("..");
114+
115+
const compiler = webpack({
116+
mode: "production",
117+
context: CONTEXT,
118+
optimization: { minimize: false },
119+
...options
120+
});
121+
compiler.outputFileSystem =
122+
/** @type {import("../").OutputFileSystem} */
123+
(/** @type {unknown} */ (createFsFromVolume(new Volume())));
124+
compiler.run((err, stats) => {
125+
if (err) return reject(err);
126+
if (!stats) return reject(new Error("no stats"));
127+
if (stats.hasErrors()) {
128+
return reject(new Error(stats.toString({ errors: true })));
129+
}
130+
resolve({
131+
compiler,
132+
stats,
133+
close: () =>
134+
new Promise((res, rej) => {
135+
compiler.close((e) => (e ? rej(e) : res()));
136+
})
137+
});
138+
});
139+
});
140+
141+
/**
142+
* Runs one watch build and resolves once the compiler has gone idle.
143+
* @param {import("../").Configuration} options webpack options
144+
* @returns {Promise<{ compiler: Compiler, stats: Stats, close: () => Promise<void> }>} the finished build
145+
*/
146+
const watchOnce = (options) =>
147+
new Promise((resolve, reject) => {
148+
const webpack = require("..");
149+
150+
const compiler = webpack({
151+
mode: "production",
152+
context: CONTEXT,
153+
optimization: { minimize: false },
154+
...options
155+
});
156+
compiler.outputFileSystem =
157+
/** @type {import("../").OutputFileSystem} */
158+
(/** @type {unknown} */ (createFsFromVolume(new Volume())));
159+
let settled = false;
160+
const watching = /** @type {import("../").Watching} */ (
161+
compiler.watch({ aggregateTimeout: 10 }, (err, stats) => {
162+
if (settled) return;
163+
settled = true;
164+
if (err) return reject(err);
165+
if (!stats) return reject(new Error("no stats"));
166+
if (stats.hasErrors()) {
167+
return reject(new Error(stats.toString({ errors: true })));
168+
}
169+
// afterDone fires after this handler on the same stack
170+
setImmediate(() => {
171+
resolve({
172+
compiler,
173+
stats,
174+
close: () =>
175+
new Promise((res, rej) => {
176+
watching.close((e) => (e ? rej(e) : res()));
177+
})
178+
});
179+
});
180+
})
181+
);
182+
});
183+
184+
describe("Compilation release", () => {
185+
it("cleans up modules of nested child compilations, not only direct children", async () => {
186+
const { compiler, stats, close } = await build({
187+
entry: "./a.js",
188+
output: { path: "/" },
189+
plugins: [nestedChildPlugin(3)]
190+
});
191+
const levels = collectLevels(stats.compilation, 0, []);
192+
expect(levels).toHaveLength(4);
193+
194+
// what the next build does before it starts
195+
const internalCompiler =
196+
/** @type {import("../lib/Compiler")} */
197+
(/** @type {unknown} */ (compiler));
198+
199+
internalCompiler._cleanupLastCompilation();
200+
201+
const retained = levels.map(({ level, modules }) => ({
202+
level,
203+
retained: retainedModules(modules)
204+
}));
205+
expect(retained).toEqual([
206+
{ level: 0, retained: [] },
207+
{ level: 1, retained: [] },
208+
{ level: 2, retained: [] },
209+
{ level: 3, retained: [] }
210+
]);
211+
await close();
212+
});
213+
214+
it("releases code generation results once a watch build goes idle", async () => {
215+
const { compiler, stats, close } = await watchOnce({
216+
entry: "./c.js",
217+
output: { path: "/" }
218+
});
219+
const internalCompiler =
220+
/** @type {import("../lib/Compiler")} */
221+
(/** @type {unknown} */ (compiler));
222+
223+
expect(stats.compilation.modules.size).toBeGreaterThan(0);
224+
expect(internalCompiler._lastCompilation).toBe(stats.compilation);
225+
const { codeGenerationResults } = stats.compilation;
226+
expect(codeGenerationResults && codeGenerationResults.map.size).toBe(0);
227+
// the stats a watch handler was handed must still render
228+
const assets = stats.toJson({ all: false, assets: true }).assets;
229+
expect(assets && assets.length).toBeGreaterThan(0);
230+
await close();
231+
});
232+
});

0 commit comments

Comments
 (0)