Skip to content

Commit df4e417

Browse files
perf(externals): match string runs with one set lookup (#22933)
* perf(externals): match string runs with one set lookup Merge each run of strings in an externals array into a set, so a node target no longer walks its 56 built-in names for every dependency, and memoize the external module identifier that concatenation asks for once per reference. * fix(externals): regroup an externals array that changed while matching Keep a copy of the entries each grouping was built from. Regroup when the array differs at the start of a match, and continue over the live array when an entry that runs user code changed it mid-walk. * fix(externals): recheck the grouping after a RegExp entry too A RegExp subclass can override test and change the externals array, so only a set of strings skips the check.
1 parent eba6cf0 commit df4e417

5 files changed

Lines changed: 171 additions & 6 deletions

File tree

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"webpack": patch
3+
---
4+
5+
Speed up externals matching, including the built-in modules of node targets.

‎lib/externals/ExternalModule.js‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -968,6 +968,8 @@ class ExternalModule extends Module {
968968
this.interop = interop;
969969
/** @type {boolean | undefined} declared side-effects state (undefined = assume side effects, webpack can't analyze an external) */
970970
this.sideEffects = sideEffects;
971+
/** @type {string | undefined} */
972+
this._identifier = undefined;
971973
if (sideEffects !== undefined) {
972974
this.factoryMeta = { sideEffectFree: !sideEffects };
973975
}
@@ -1038,6 +1040,8 @@ class ExternalModule extends Module {
10381040
* @returns {string} a unique identifier of the module
10391041
*/
10401042
identifier() {
1043+
// Concatenation asks once per reference, and building it stringifies the request.
1044+
if (this._identifier !== undefined) return this._identifier;
10411045
let id = `external ${this._resolveExternalType(
10421046
this.externalType
10431047
)} ${JSON.stringify(this.request)}`;
@@ -1058,7 +1062,7 @@ class ExternalModule extends Module {
10581062
if (this.sideEffects !== undefined) {
10591063
id += `|sideEffects=${this.sideEffects}`;
10601064
}
1061-
return id;
1065+
return (this._identifier = id);
10621066
}
10631067

10641068
/**

‎lib/externals/ExternalModuleFactoryPlugin.js‎

Lines changed: 111 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,38 @@ const OPTIONS_FORM_KEYS = new Set(["external", "sideEffects"]);
129129

130130
const PLUGIN_NAME = "ExternalModuleFactoryPlugin";
131131

132+
/**
133+
* An externals array with each run of strings merged into one set.
134+
* @typedef {object} GroupedExternals
135+
* @property {Externals[]} source the entries the grouping was built from
136+
* @property {(Externals | Set<string>)[]} entries the entries, each run of strings as one set
137+
* @property {number[]} ends for each entry, the index in the array just past it
138+
*/
139+
140+
/**
141+
* Whether matching an externals entry can call user code, which may change the array.
142+
* Only a set of strings is matched without it: a RegExp may override `test`, an object may hold a `byLayer` function.
143+
* @param {Externals | Set<string>} entry a grouped entry
144+
* @returns {boolean} true unless the entry is a set of strings
145+
*/
146+
const mayRunUserCode = (entry) => !(entry instanceof Set);
147+
148+
/**
149+
* Whether an externals array still holds the entries it was grouped from.
150+
* An externals function may change the array while requests are matched.
151+
* @param {Externals[]} externals externals array
152+
* @param {GroupedExternals} grouped its grouping
153+
* @returns {boolean} true when the grouping is current
154+
*/
155+
const isCurrentGrouping = (externals, grouped) => {
156+
const { source } = grouped;
157+
if (externals.length !== source.length) return false;
158+
for (let i = 0; i < source.length; i++) {
159+
if (externals[i] !== source[i]) return false;
160+
}
161+
return true;
162+
};
163+
132164
class ExternalModuleFactoryPlugin {
133165
/**
134166
* Creates an instance of ExternalModuleFactoryPlugin.
@@ -148,6 +180,47 @@ class ExternalModuleFactoryPlugin {
148180
*/
149181
apply(normalModuleFactory) {
150182
const globalType = this.type;
183+
/** @type {WeakMap<Externals[], GroupedExternals>} */
184+
const groupedArrays = new WeakMap();
185+
186+
/**
187+
* Merges each run of request strings in an externals array into one set.
188+
* A string entry always externalizes the request as written, so which string of a run matched never matters.
189+
* @param {Externals[]} externals externals array
190+
* @returns {GroupedExternals} the entries with each run of strings as a set, regrouped when the array changed
191+
*/
192+
const getGroupedArray = (externals) => {
193+
const cached = groupedArrays.get(externals);
194+
if (cached !== undefined && isCurrentGrouping(externals, cached)) {
195+
return cached;
196+
}
197+
/** @type {GroupedExternals["entries"]} */
198+
const entries = [];
199+
/** @type {number[]} */
200+
const ends = [];
201+
/** @type {Set<string> | undefined} */
202+
let strings;
203+
for (let i = 0; i < externals.length; i++) {
204+
const item = externals[i];
205+
if (typeof item === "string") {
206+
if (strings === undefined) {
207+
strings = new Set();
208+
entries.push(strings);
209+
ends.push(i + 1);
210+
}
211+
strings.add(item);
212+
ends[ends.length - 1] = i + 1;
213+
} else {
214+
strings = undefined;
215+
entries.push(item);
216+
ends.push(i + 1);
217+
}
218+
}
219+
/** @type {GroupedExternals} */
220+
const grouped = { source: [...externals], entries, ends };
221+
groupedArrays.set(externals, grouped);
222+
return grouped;
223+
};
151224
normalModuleFactory.hooks.factorize.tapAsync(
152225
PLUGIN_NAME,
153226
(data, callback) => {
@@ -162,6 +235,7 @@ class ExternalModuleFactoryPlugin {
162235
? dependency.originalRequest
163236
: request;
164237
const hasOriginalRequest = originalRequest !== request;
238+
const alternateRequest = getAlternateCoreModuleRequest(request);
165239

166240
/** @typedef {(err?: Error | null, externalModule?: ExternalModule) => void} HandleExternalCallback */
167241

@@ -322,12 +396,16 @@ class ExternalModuleFactoryPlugin {
322396
if (
323397
externals === request ||
324398
(hasOriginalRequest && externals === originalRequest) ||
325-
externals === getAlternateCoreModuleRequest(request)
399+
externals === alternateRequest
326400
) {
327401
return handleExternal(originalRequest, undefined, callback);
328402
}
329403
} else if (Array.isArray(externals)) {
330-
let i = 0;
404+
const grouped = getGroupedArray(externals);
405+
const { entries, ends } = grouped;
406+
let groupIndex = 0;
407+
// The index into `externals` once a non-string entry changed the array.
408+
let liveIndex = -1;
331409
const next = () => {
332410
/** @type {boolean | undefined} */
333411
let asyncFlag;
@@ -351,8 +429,37 @@ class ExternalModuleFactoryPlugin {
351429

352430
do {
353431
asyncFlag = true;
354-
if (i >= externals.length) return callback();
355-
handleExternals(externals[i++], handleExternalsAndCallback);
432+
if (
433+
liveIndex === -1 &&
434+
groupIndex > 0 &&
435+
mayRunUserCode(entries[groupIndex - 1]) &&
436+
!isCurrentGrouping(externals, grouped)
437+
) {
438+
liveIndex = ends[groupIndex - 1];
439+
}
440+
if (liveIndex !== -1) {
441+
if (liveIndex >= externals.length) return callback();
442+
handleExternals(
443+
externals[liveIndex++],
444+
handleExternalsAndCallback
445+
);
446+
continue;
447+
}
448+
if (groupIndex >= entries.length) return callback();
449+
const item = entries[groupIndex++];
450+
if (item instanceof Set) {
451+
if (
452+
item.has(request) ||
453+
(hasOriginalRequest && item.has(originalRequest)) ||
454+
(alternateRequest !== undefined &&
455+
item.has(alternateRequest))
456+
) {
457+
return handleExternal(originalRequest, undefined, callback);
458+
}
459+
asyncFlag = false;
460+
} else {
461+
handleExternals(item, handleExternalsAndCallback);
462+
}
356463
} while (!asyncFlag);
357464
asyncFlag = false;
358465
};
@@ -470,7 +577,6 @@ class ExternalModuleFactoryPlugin {
470577
);
471578
}
472579
// Fall back to the `node:`-prefixed/unprefixed core module form.
473-
const alternateRequest = getAlternateCoreModuleRequest(request);
474580
if (
475581
alternateRequest !== undefined &&
476582
Object.prototype.hasOwnProperty.call(
Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
it("should keep the order of string entries in an externals array", () => {
2+
const os = require("os");
3+
expect(typeof os.platform).toBe("function");
4+
const path = require("path");
5+
expect(typeof path.join).toBe("function");
6+
expect(path.platform).toBeUndefined();
7+
expect(require("node:path")).toBe(path);
8+
expect(typeof require("util").inspect).toBe("function");
9+
expect(require("os-alias")).toBe(os);
10+
expect(typeof require("zlib").gzipSync).toBe("function");
11+
});
Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
"use strict";
2+
3+
/** @type {import("../../../../").ExternalItem[]} */
4+
const externals = [
5+
"path",
6+
"util",
7+
// Runs between two runs of strings, so it must still win for "os-alias".
8+
({ request }, callback) => {
9+
if (request === "os-alias") return callback(null, "os");
10+
// Appends an entry while matching: this walk and later ones must see it.
11+
if (request === "os" && !externals.includes("os")) externals.push("os");
12+
// Replaces an entry in place while matching a request nothing externalizes.
13+
if (request.startsWith("./index")) externals[4] = { path: "os" };
14+
callback();
15+
},
16+
"os-alias",
17+
// The strings above match "path" first, so this mapping never applies.
18+
{ path: "os" },
19+
// A RegExp subclass runs user code too, and may append an entry the same way.
20+
new (class extends RegExp {
21+
/**
22+
* @param {string} request the request
23+
* @returns {boolean} never a match itself
24+
*/
25+
test(request) {
26+
if (request === "zlib" && !externals.includes("zlib")) {
27+
externals.push("zlib");
28+
}
29+
return false;
30+
}
31+
})("^$")
32+
];
33+
34+
/** @type {import("../../../../").Configuration} */
35+
module.exports = {
36+
target: "web",
37+
externalsType: "commonjs",
38+
externals
39+
};

0 commit comments

Comments
 (0)