Skip to content

Commit c7d8a3a

Browse files
alexander-akaitalexander-akait
authored
fix: release per-child Compilation heap pressure in MultiCompiler (#21015)
1 parent d6cdebe commit c7d8a3a

5 files changed

Lines changed: 137 additions & 3 deletions

File tree

‎.changeset/multicompiler-release-per-child-memory.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"webpack": patch
3+
---
4+
5+
Release per-child `codeGenerationResults` in `MultiCompiler` and at `Compiler.close` to reduce memory retention.

‎lib/Compiler.js‎

Lines changed: 27 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -491,6 +491,25 @@ class Compiler {
491491
}
492492
}
493493

494+
/**
495+
* Release fields on a finished compilation that nothing reads after emit,
496+
* so the heap can shrink while user code still holds the Stats reference.
497+
* Recurses into child compilations. Stats output is preserved — only
498+
* codeGen byproducts are dropped.
499+
* @param {Compilation} compilation finished compilation to slim down
500+
* @returns {void}
501+
*/
502+
_releaseUnusedCompilationData(compilation) {
503+
for (const child of compilation.children) {
504+
this._releaseUnusedCompilationData(child);
505+
}
506+
// Rendered source per (module × runtime) — used only during seal/emit,
507+
// never read by Stats, and not serialized to the persistent cache.
508+
if (compilation.codeGenerationResults !== undefined) {
509+
compilation.codeGenerationResults.map.clear();
510+
}
511+
}
512+
494513
/**
495514
* Returns a compiler watcher.
496515
* @param {WatchOptions} watchOptions the watcher's options
@@ -1439,9 +1458,14 @@ ${other}`);
14391458
}
14401459
this.hooks.shutdown.callAsync((err) => {
14411460
if (err) return callback(err);
1442-
// Get rid of reference to last compilation to avoid leaking memory
1443-
// We can't run this._cleanupLastCompilation() as the Stats to this compilation
1444-
// might be still in use. We try to get rid of the reference to the cache instead.
1461+
// Defer a microtask so a close() made inside the run callback can't
1462+
// release codeGenerationResults before afterDone fires on the same stack.
1463+
const lastCompilation = this._lastCompilation;
1464+
if (lastCompilation !== undefined) {
1465+
Promise.resolve().then(() => {
1466+
this._releaseUnusedCompilationData(lastCompilation);
1467+
});
1468+
}
14451469
this._lastCompilation = undefined;
14461470
this._lastNormalModuleFactory = undefined;
14471471
this.cache.shutdown(callback);

‎lib/MultiCompiler.js‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,20 @@ module.exports = class MultiCompiler {
128128
doneCompilers--;
129129
}
130130
});
131+
// Release fields on this child's Compilation once it's done. The
132+
// stage: Infinity tap runs after every afterDone tap at a lower
133+
// stage, so plugins observing compilation state in afterDone still
134+
// see it intact. Stats remains usable; only fields Stats never reads
135+
// (and that the persistent cache never serializes) are dropped.
136+
137+
compiler.hooks.afterDone.tap(
138+
{ name: CLASS_NAME, stage: Infinity },
139+
(stats) => {
140+
if (stats !== undefined) {
141+
compiler._releaseUnusedCompilationData(stats.compilation);
142+
}
143+
}
144+
);
131145
}
132146
this._validateCompilersOptions();
133147
}

‎test/Compiler.test.js‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -326,6 +326,42 @@ describe("Compiler", () => {
326326
compiler.close(done);
327327
});
328328

329+
it("should release codeGenerationResults on close while Stats stays usable and afterDone still sees them (#15521)", (done) => {
330+
const webpack = require("..");
331+
332+
const compiler = webpack({
333+
context: path.join(__dirname, "fixtures"),
334+
mode: "production",
335+
entry: "./c",
336+
output: {
337+
path: "/directory",
338+
filename: "bundle.js"
339+
}
340+
});
341+
compiler.outputFileSystem = createFsFromVolume(new Volume());
342+
let sizeSeenByAfterDone;
343+
compiler.hooks.afterDone.tap("Test", (stats) => {
344+
sizeSeenByAfterDone = stats.compilation.codeGenerationResults.map.size;
345+
});
346+
compiler.run((err, stats) => {
347+
if (err) return done(err);
348+
const { compilation } = stats;
349+
expect(compilation.codeGenerationResults.map.size).toBeGreaterThan(0);
350+
// close() runs inside the run callback, i.e. before Compiler.run fires
351+
// afterDone. The release is deferred a microtask, so afterDone still
352+
// observes the results; assert via setTimeout once the defer ran.
353+
compiler.close((closeErr) => {
354+
if (closeErr) return done(closeErr);
355+
setTimeout(() => {
356+
expect(sizeSeenByAfterDone).toBeGreaterThan(0);
357+
expect(compilation.codeGenerationResults.map.size).toBe(0);
358+
expect(typeof stats.toJson().hash).toBe("string");
359+
done();
360+
}, 0);
361+
});
362+
});
363+
});
364+
329365
it("should not emit on errors", (done) => {
330366
const webpack = require("..");
331367

‎test/MultiCompiler.test.js‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,61 @@ describe("MultiCompiler", () => {
141141
});
142142
});
143143

144+
it("should release per-child compilation memory as each child finishes (#15521)", (done) => {
145+
const compiler = createMultiCompiler();
146+
compiler.run((err, stats) => {
147+
if (err) return done(err);
148+
for (const childStats of stats.stats) {
149+
const compilation = childStats.compilation;
150+
// codeGenerationResults: only used during seal/emit, dropped.
151+
expect(compilation.codeGenerationResults.map.size).toBe(0);
152+
// Stats must still be usable on the slimmed compilation.
153+
expect(typeof childStats.toJson().hash).toBe("string");
154+
}
155+
compiler.close(done);
156+
});
157+
});
158+
159+
it("should release a finished child's codeGenerationResults before a dependent sibling runs (#15521)", (done) => {
160+
const compiler = webpack(
161+
Object.assign(
162+
[
163+
{
164+
name: "a",
165+
context: path.join(__dirname, "fixtures"),
166+
entry: "./a.js"
167+
},
168+
{
169+
name: "b",
170+
context: path.join(__dirname, "fixtures"),
171+
entry: "./b.js",
172+
dependencies: ["a"]
173+
}
174+
],
175+
{ parallelism: 1 }
176+
)
177+
);
178+
compiler.outputFileSystem = createFsFromVolume(new Volume());
179+
compiler.watchFileSystem = { watch(_a, _b, _c, _d, _e, _f, _g) {} };
180+
const [a, b] = compiler.compilers;
181+
let aCompilation;
182+
a.hooks.done.tap("test", (stats) => {
183+
aCompilation = stats.compilation;
184+
});
185+
// With dependencies + parallelism 1, b only starts after a is fully
186+
// done (including a's afterDone release tap). Capture a's map size at
187+
// that point: it must already be cleared while b is about to build.
188+
let aMapSizeWhenBStarts;
189+
b.hooks.run.tap("test", () => {
190+
aMapSizeWhenBStarts = aCompilation.codeGenerationResults.map.size;
191+
});
192+
compiler.run((err) => {
193+
if (err) return done(err);
194+
expect(aMapSizeWhenBStarts).toBe(0);
195+
compiler.close(done);
196+
});
197+
});
198+
144199
it("should watch again correctly after first compilation", (done) => {
145200
const compiler = createMultiCompiler();
146201
compiler.run((err, _stats) => {

0 commit comments

Comments
 (0)