Skip to content

Commit 35dad2f

Browse files
perf(concatenation): release each module's ast after scope analysis
Concatenation and the inlined-entry rename kept every analysed module's whole ast alive until code generation finished, for one lookup each: is this identifier the value of a shorthand property? The analyser now records where each shorthand property's identifier starts, so the rename reads an offset instead of walking the tree, and neither caller keeps the ast. A scope kept its opening node too, which held the tree by itself; it now keeps that node's type and offset. Over a 1007-module concatenation (three.js + lodash-es, 242k ast nodes), the GC-stabilized heap at the end of analysis drops from 87.44 MB to 71.06 MB. getPathInAst had no other caller and is removed with its helpers.
1 parent 107ebef commit 35dad2f

13 files changed

Lines changed: 204 additions & 496 deletions

File tree

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"webpack": patch
3+
---
4+
5+
Release each module's ast once it is scope-analysed for concatenation.

‎lib/javascript/JavascriptModulesPlugin.js‎

Lines changed: 15 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,6 @@ const {
4444
addScopeSymbols,
4545
findNewName,
4646
getAllReferences,
47-
getPathInAst,
4847
getUsedNamesInScopeInfo
4948
} = require("../util/concatenate");
5049
const createHash = require("../util/createHash");
@@ -60,11 +59,6 @@ const analyzeScope = require("./ScopeAnalyzer");
6059
const { renderAwaitedEntries } = require("./StartupHelpers");
6160
const { releaseParserCaches } = require("./syntax-parser");
6261

63-
/**
64-
* @import {
65-
* Program
66-
* } from "estree"
67-
*/
6862
/** @import { Source } from "webpack-sources" */
6963
/**
7064
* @import {
@@ -363,7 +357,7 @@ const createCompilationHooks = () => ({
363357
* in `allUsedNames`. Lets inlined modules be emitted without a per-entry/per-chunk
364358
* IIFE by resolving name collisions instead of isolating each module.
365359
* @param {InstanceType<typeof import("webpack-sources").ReplaceSource>} source the module source to edit
366-
* @param {Program} ast the parsed module AST
360+
* @param {Set<number>} shorthandIdentifierStarts where each shorthand property's identifier starts
367361
* @param {Variable} variable the top-level variable to check/rename
368362
* @param {string} moduleIdentifier the module identifier (for scope caching)
369363
* @param {string} readableIdentifier the readable module identifier (for new names)
@@ -373,7 +367,7 @@ const createCompilationHooks = () => ({
373367
*/
374368
const renameInlinedModuleVariable = (
375369
source,
376-
ast,
370+
shorthandIdentifierStarts,
377371
variable,
378372
moduleIdentifier,
379373
readableIdentifier,
@@ -412,16 +406,9 @@ const renameInlinedModuleVariable = (
412406
renamedDeclarations.set(name, newName);
413407
for (const identifier of allIdentifiers) {
414408
const r = /** @type {Range} */ (identifier.range);
415-
const path = getPathInAst(ast, identifier);
416-
if (path && path.length > 1) {
417-
const maybeProperty =
418-
path[1].type === "AssignmentPattern" && path[1].left === path[0]
419-
? path[2]
420-
: path[1];
421-
if (maybeProperty.type === "Property" && maybeProperty.shorthand) {
422-
source.insert(r[1], `: ${newName}`);
423-
continue;
424-
}
409+
if (shorthandIdentifierStarts.has(r[0])) {
410+
source.insert(r[1], `: ${newName}`);
411+
continue;
425412
}
426413
source.replace(r[0], r[1] - 1, newName);
427414
}
@@ -2163,7 +2150,7 @@ class JavascriptModulesPlugin {
21632150
const renamedDeclarations = new Map();
21642151
const { runtimeTemplate } = renderContext;
21652152

2166-
/** @typedef {{ source: Source, module: Module, ast: Program, variables: Set<Variable>, through: Set<Reference>, usedInNonInlined: Set<Variable>, moduleScope: Scope }} Info */
2153+
/** @typedef {{ source: Source, module: Module, shorthandIdentifierStarts: Set<number>, variables: Set<Variable>, through: Set<Reference>, usedInNonInlined: Set<Variable>, moduleScope: Scope }} Info */
21672154
/** @type {Map<Module, Info>} */
21682155
const inlinedModulesToInfo = new Map();
21692156
/** @type {Set<string>} */
@@ -2296,7 +2283,7 @@ class JavascriptModulesPlugin {
22962283
const moduleScope = analysis.moduleScope;
22972284
inlinedModulesToInfo.set(m, {
22982285
source: moduleSource,
2299-
ast,
2286+
shorthandIdentifierStarts: analysis.shorthandIdentifierStarts,
23002287
module: m,
23012288
variables: new Set(moduleScope.variables),
23022289
through: new Set(analysis.unresolvedReferences),
@@ -2327,14 +2314,14 @@ class JavascriptModulesPlugin {
23272314
for (const ref of through) allUsedNames.add(ref.identifier.name);
23282315
}
23292316
for (const [m, moduleInfo] of inlinedModulesToInfo) {
2330-
const { ast, source: _source } = moduleInfo;
2317+
const { shorthandIdentifierStarts, source: _source } = moduleInfo;
23312318
const source = new ReplaceSource(_source);
23322319
/** @type {Map<string, string>} */
23332320
const renamed = new Map();
23342321
for (const variable of moduleInfo.variables) {
23352322
renameInlinedModuleVariable(
23362323
source,
2337-
ast,
2324+
shorthandIdentifierStarts,
23382325
variable,
23392326
moduleInfo.module.identifier(),
23402327
m.readableIdentifier(runtimeTemplate.requestShortener),
@@ -2352,7 +2339,11 @@ class JavascriptModulesPlugin {
23522339
}
23532340

23542341
for (const [m, moduleInfo] of inlinedModulesToInfo) {
2355-
const { ast, source: _source, usedInNonInlined } = moduleInfo;
2342+
const {
2343+
shorthandIdentifierStarts,
2344+
source: _source,
2345+
usedInNonInlined
2346+
} = moduleInfo;
23562347
const source = new ReplaceSource(_source);
23572348
if (usedInNonInlined.size === 0) {
23582349
renamedInlinedModules.set(m, source);
@@ -2373,7 +2364,7 @@ class JavascriptModulesPlugin {
23732364
for (const variable of info.variables) {
23742365
renameInlinedModuleVariable(
23752366
source,
2376-
ast,
2367+
shorthandIdentifierStarts,
23772368
variable,
23782369
info.module.identifier(),
23792370
m.readableIdentifier(runtimeTemplate.requestShortener),

‎lib/javascript/ScopeAnalyzer.js‎

Lines changed: 36 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,8 @@
1010
* Function as ESTreeFunction,
1111
* Identifier,
1212
* Node,
13-
* Program
13+
* Program,
14+
* Property
1415
* } from "estree"
1516
*/
1617

@@ -116,16 +117,18 @@ const CHILD_KEYS = {
116117
class Scope {
117118
/**
118119
* @param {ScopeType} type what opened this scope
119-
* @param {Node} block the node that opened this scope
120+
* @param {Node} block the node that opened this scope, read but not kept
120121
* @param {Scope | null} upper the enclosing scope
121122
* @param {boolean} isVarScope whether `var` and function declarations hoist to here
122123
* @param {boolean} recordEveryReference whether every binding collects its references, not only the ones webpack reads
123124
*/
124125
constructor(type, block, upper, isVarScope, recordEveryReference) {
125126
/** @type {ScopeType} */
126127
this.type = type;
127-
/** @type {Node} */
128-
this.block = block;
128+
/** @type {string} the type of the node that opened this scope */
129+
this.blockType = block.type;
130+
/** @type {number} where that node starts */
131+
this.blockStart = /** @type {Node & Offset} */ (block).start;
129132
/** @type {Scope | null} */
130133
this.upper = upper;
131134
/** @type {Scope[]} */
@@ -389,10 +392,26 @@ class ScopeAnalyzer {
389392
this.pendingIdentifiers = [];
390393
/** @type {Scope[]} the scope each pending identifier was seen in */
391394
this.pendingScopes = [];
395+
/** @type {Set<number>} where each shorthand property's identifier starts */
396+
this.shorthandIdentifierStarts = new Set();
392397
/** @type {boolean} */
393398
this.recordEveryReference = recordEveryReference;
394399
}
395400

401+
/**
402+
* Records where a shorthand property's identifier starts, so renaming it
403+
* later needs the offset rather than the tree it was read from.
404+
* @param {Property} property a property whose `shorthand` is set
405+
* @returns {void}
406+
*/
407+
_recordShorthandIdentifier(property) {
408+
const value = property.value;
409+
const identifier = value.type === "AssignmentPattern" ? value.left : value;
410+
this.shorthandIdentifierStarts.add(
411+
/** @type {Identifier & Offset} */ (identifier).start
412+
);
413+
}
414+
396415
/**
397416
* @param {ScopeType} type scope kind
398417
* @param {Node} block node opening the scope
@@ -512,6 +531,7 @@ class ScopeAnalyzer {
512531
}
513532
return;
514533
case "Property":
534+
if (node.shorthand) this._recordShorthandIdentifier(node);
515535
if (node.computed) rightHandNodes.push(node.key);
516536
this._patternWalk(node.value, mode, target, defaults, rightHandNodes);
517537
return;
@@ -794,6 +814,11 @@ class ScopeAnalyzer {
794814
return;
795815

796816
case "Property":
817+
if (node.shorthand) this._recordShorthandIdentifier(node);
818+
if (node.computed) this._visit(node.key);
819+
this._visit(node.value);
820+
return;
821+
797822
case "MethodDefinition":
798823
if (node.computed) this._visit(node.key);
799824
this._visit(node.value);
@@ -1068,6 +1093,7 @@ class ScopeAnalyzer {
10681093
* @property {Scope} globalScope the outermost scope
10691094
* @property {Scope} moduleScope the module body scope, where top-level declarations live
10701095
* @property {Reference[]} unresolvedReferences every identifier that resolved to no binding — the module's free names
1096+
* @property {Set<number>} shorthandIdentifierStarts where each shorthand property's identifier starts, so a rename can write `name: newName` without the tree
10711097
*/
10721098

10731099
/**
@@ -1084,7 +1110,12 @@ const analyzeScope = (ast, recordEveryReference = false) => {
10841110
const moduleScope = analyzer._push("module", ast, true);
10851111
analyzer._visitAll(ast.body);
10861112
const unresolvedReferences = analyzer._resolve();
1087-
return { globalScope, moduleScope, unresolvedReferences };
1113+
return {
1114+
globalScope,
1115+
moduleScope,
1116+
unresolvedReferences,
1117+
shorthandIdentifierStarts: analyzer.shorthandIdentifierStarts
1118+
};
10881119
};
10891120

10901121
analyzeScope.Reference = Reference;

‎lib/optimize/ConcatenatedModule.js‎

Lines changed: 11 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,6 @@ const {
4848
addScopeSymbols,
4949
findNewName,
5050
getAllReferences,
51-
getPathInAst,
5251
getUsedNamesInScopeInfo,
5352
isCommonJsConcatenationEnabled
5453
} = require("../util/concatenate");
@@ -205,7 +204,7 @@ const findIdentifiers = (source) => {
205204
* @property {boolean} eager a wrapped body reached by an eager import edge, so its accessor is called at its own slot in evaluation order
206205
* @property {Module} module
207206
* @property {number} index
208-
* @property {Program | undefined} ast
207+
* @property {Set<number> | undefined} shorthandIdentifierStarts where each shorthand property's identifier starts
209208
* @property {Source | undefined} internalSource
210209
* @property {ReplaceSource | undefined} source
211210
* @property {InitFragment<ChunkRenderContext>[]=} chunkInitFragments
@@ -1710,7 +1709,7 @@ class ConcatenatedModule extends Module {
17101709
// | (concatenated [+wrapped] | external | reference)
17111710
// v
17121711
// [1] ANALYSE — inner codeGeneration() per module, into a ConcatenationScope
1713-
// | plain -> ReplaceSource + ast + scope analysis
1712+
// | plain -> ReplaceSource + scope analysis
17141713
// | wrapped -> ReplaceSource only, body kept verbatim
17151714
// | external -> nothing yet, rendered as a require() below
17161715
// [2] RESERVE — free globals of the analysed bodies enter allUsedNames, so
@@ -1720,7 +1719,7 @@ class ConcatenatedModule extends Module {
17201719
// | the generated names allocated: wrapper accessor, namespace object,
17211720
// | interop helpers, external import variable
17221721
// [4] LINK — __WEBPACK_MODULE_REFERENCE__ tokens replaced by their final
1723-
// | binding; through the ast for analysed bodies, textually for wrapped
1722+
// | binding; through the scopes for analysed bodies, textually for wrapped
17241723
// [5] COLLECT EXPORTS — root exports split into used / unused / inlined
17251724
// v
17261725
// ---- from here on everything is appended to the result ConcatSource ----
@@ -2144,26 +2143,14 @@ class ConcatenatedModule extends Module {
21442143
for (const identifier of variable.identifiers) {
21452144
allIdentifiers.add(identifier);
21462145
}
2146+
const shorthandStarts =
2147+
/** @type {NonNullable<ConcatenatedModuleInfo["shorthandIdentifierStarts"]>} */
2148+
(info.shorthandIdentifierStarts);
21472149
for (const identifier of allIdentifiers) {
21482150
const r = /** @type {Range} */ (identifier.range);
2149-
const path = getPathInAst(
2150-
/** @type {NonNullable<ConcatenatedModuleInfo["ast"]>} */
2151-
(info.ast),
2152-
identifier
2153-
);
2154-
if (path && path.length > 1) {
2155-
const maybeProperty =
2156-
path[1].type === "AssignmentPattern" &&
2157-
path[1].left === path[0]
2158-
? path[2]
2159-
: path[1];
2160-
if (
2161-
maybeProperty.type === "Property" &&
2162-
maybeProperty.shorthand
2163-
) {
2164-
source.insert(r[1], `: ${newName}`);
2165-
continue;
2166-
}
2151+
if (shorthandStarts.has(r[0])) {
2152+
source.insert(r[1], `: ${newName}`);
2153+
continue;
21672154
}
21682155
source.replace(r[0], r[1] - 1, newName);
21692156
}
@@ -3151,7 +3138,7 @@ ${defineGetters}`
31513138
// the wrapper isolates the body's declarations, so [3] renames nothing in
31523139
// it and its scopes are not kept; [2] claims its identifiers textually
31533140
if (info.wrapped) return;
3154-
info.ast = ast;
3141+
info.shorthandIdentifierStarts = analysis.shorthandIdentifierStarts;
31553142
info.globalScope = analysis.globalScope;
31563143
info.moduleScope = analysis.moduleScope;
31573144
} catch (err) {
@@ -3188,7 +3175,7 @@ ${defineGetters}`
31883175
module: info.module,
31893176
index,
31903177
namespaceFnName: undefined,
3191-
ast: undefined,
3178+
shorthandIdentifierStarts: undefined,
31923179
chunkInitFragments: undefined,
31933180
internalSource: undefined,
31943181
runtimeRequirements: undefined,

0 commit comments

Comments
 (0)