Skip to content

Commit 267ef49

Browse files
authored
perf(watch): share one watcher pool between MultiCompiler children (#21641)
* perf(watch): share one watcher pool between MultiCompiler children Watchpack pools its DirectoryWatchers per options object identity, and every Watching builds its own options object, so a MultiCompiler's children never shared one. Each child rescanned (readdir + lstat) every watched directory it had in common with the others. Intern the options object by value so equal options reach watchpack as the same reference. On 3000 modules with 4 configs: 860 -> 215 DirectoryWatchers, 12060 -> 3015 tracked entries, 22.8 -> 15.9 MB of watcher heap. Children that watch differently still get their own pool. * fix: snapshot the canonical watch options instead of the caller's object The pool entry kept the first caller's options object, so mutating it afterwards handed the next caller options it never asked for: a child asking for an aggregateTimeout of 20 got whatever the earlier child had since changed it to.
1 parent 40e14f9 commit 267ef49

3 files changed

Lines changed: 179 additions & 1 deletion

File tree

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"webpack": patch
3+
---
4+
5+
Share one file watcher pool between MultiCompiler children.

‎lib/node/NodeWatchFileSystem.js‎

Lines changed: 55 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,60 @@ const Watchpack = require("watchpack");
1616
/** @typedef {import("../util/fs").Changes} Changes */
1717
/** @typedef {import("../util/fs").Removals} Removals */
1818

19+
// Watchpack pools DirectoryWatchers per options object identity; interning by
20+
// value lets a MultiCompiler's children share one pool instead of one scan each.
21+
/** @type {Map<string, WatchOptions>} */
22+
const canonicalWatchOptions = new Map();
23+
/** @type {WeakMap<RegExp, string>} */
24+
const regExpKeys = new WeakMap();
25+
let nextRegExpKey = 0;
26+
27+
// A RegExp keys by reference, so two configs spelling the same one inline do
28+
// not share a pool.
29+
/**
30+
* @param {WatchOptions["ignored"]} ignored the `ignored` option
31+
* @returns {string} key describing it, or "" when it cannot be keyed
32+
*/
33+
const ignoredKey = (ignored) => {
34+
if (ignored === undefined || ignored === null) return "-";
35+
if (typeof ignored === "string") return `s${ignored}`;
36+
if (Array.isArray(ignored)) return `a${JSON.stringify(ignored)}`;
37+
if (!(ignored instanceof RegExp)) return "";
38+
let key = regExpKeys.get(ignored);
39+
if (key === undefined) {
40+
key = `r${nextRegExpKey++}`;
41+
regExpKeys.set(ignored, key);
42+
}
43+
return key;
44+
};
45+
46+
// Snapshot rather than keep the caller's object: mutating it later would hand
47+
// the next caller options it never asked for.
48+
/**
49+
* @param {WatchOptions} options watch options
50+
* @returns {WatchOptions} a shared object equal to `options`, or `options`
51+
*/
52+
const getCanonicalWatchOptions = (options) => {
53+
const ignored = ignoredKey(options.ignored);
54+
if (ignored === "") return options;
55+
const key = `${options.aggregateTimeout}|${Boolean(
56+
options.followSymlinks
57+
)}|${options.poll}|${ignored}`;
58+
const canonical = canonicalWatchOptions.get(key);
59+
if (canonical !== undefined) return canonical;
60+
/** @type {WatchOptions} */
61+
const snapshot = {
62+
aggregateTimeout: options.aggregateTimeout,
63+
followSymlinks: options.followSymlinks,
64+
ignored: Array.isArray(options.ignored)
65+
? [...options.ignored]
66+
: options.ignored,
67+
poll: options.poll
68+
};
69+
canonicalWatchOptions.set(key, snapshot);
70+
return snapshot;
71+
};
72+
1973
class NodeWatchFileSystem {
2074
/**
2175
* Creates an instance of NodeWatchFileSystem.
@@ -64,7 +118,7 @@ class NodeWatchFileSystem {
64118
throw new Error("Invalid arguments: 'callbackUndelayed'");
65119
}
66120
const oldWatcher = this.watcher;
67-
this.watcher = new Watchpack(options);
121+
this.watcher = new Watchpack(getCanonicalWatchOptions(options));
68122

69123
if (callbackUndelayed) {
70124
this.watcher.once("change", callbackUndelayed);
Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
"use strict";
2+
3+
const path = require("path");
4+
const NodeWatchFileSystem = require("../lib/node/NodeWatchFileSystem");
5+
6+
const fixtures = path.join(__dirname, "fixtures");
7+
const file = path.join(fixtures, "a.js");
8+
9+
/** @type {{ close: () => void }[]} */
10+
let watchers;
11+
12+
/**
13+
* Starts a watcher over the same file with the given options.
14+
* @param {import("../lib/util/fs").WatchOptions} options watch options
15+
* @returns {InstanceType<import("watchpack")>} the watchpack instance behind it
16+
*/
17+
const watch = (options) => {
18+
const wfs = new NodeWatchFileSystem(
19+
/** @type {import("../lib/util/fs").InputFileSystem} */ ({})
20+
);
21+
watchers.push(
22+
wfs.watch(
23+
[file],
24+
[],
25+
[],
26+
Date.now(),
27+
options,
28+
() => {},
29+
() => {}
30+
)
31+
);
32+
return /** @type {InstanceType<import("watchpack")>} */ (wfs.watcher);
33+
};
34+
35+
describe("NodeWatchFileSystem", () => {
36+
beforeEach(() => {
37+
watchers = [];
38+
});
39+
40+
afterEach(() => {
41+
for (const watcher of watchers) watcher.close();
42+
});
43+
44+
// Sharing the pool is what keeps a MultiCompiler from scanning every watched
45+
// directory once per child.
46+
it("shares one watcher pool between equal options", () => {
47+
const a = watch({ aggregateTimeout: 20 });
48+
const b = watch({ aggregateTimeout: 20 });
49+
50+
expect(b.watcherManager).toBe(a.watcherManager);
51+
expect(a.watcherManager.directoryWatchers.size).toBe(1);
52+
});
53+
54+
it("keeps separate pools for different aggregateTimeout", () => {
55+
const a = watch({ aggregateTimeout: 20 });
56+
const b = watch({ aggregateTimeout: 300 });
57+
58+
expect(b.watcherManager).not.toBe(a.watcherManager);
59+
});
60+
61+
it("keeps separate pools for different ignored patterns", () => {
62+
const a = watch({ aggregateTimeout: 20, ignored: "**/a" });
63+
const b = watch({ aggregateTimeout: 20, ignored: "**/b" });
64+
const c = watch({ aggregateTimeout: 20, ignored: ["**/a"] });
65+
66+
expect(b.watcherManager).not.toBe(a.watcherManager);
67+
expect(c.watcherManager).not.toBe(a.watcherManager);
68+
});
69+
70+
it("shares equal ignored patterns given as arrays", () => {
71+
const a = watch({ aggregateTimeout: 20, ignored: ["**/a", "**/b"] });
72+
const b = watch({ aggregateTimeout: 20, ignored: ["**/a", "**/b"] });
73+
74+
expect(b.watcherManager).toBe(a.watcherManager);
75+
});
76+
77+
it("keeps separate pools for different poll and followSymlinks", () => {
78+
const a = watch({ aggregateTimeout: 20 });
79+
const b = watch({ aggregateTimeout: 20, poll: 100 });
80+
const c = watch({ aggregateTimeout: 20, followSymlinks: true });
81+
82+
expect(b.watcherManager).not.toBe(a.watcherManager);
83+
expect(c.watcherManager).not.toBe(a.watcherManager);
84+
});
85+
86+
// A RegExp can only be compared by reference, never by value.
87+
it("shares an ignored RegExp only when it is the same reference", () => {
88+
const ignored = /node_modules/;
89+
const a = watch({ aggregateTimeout: 20, ignored });
90+
const b = watch({ aggregateTimeout: 20, ignored });
91+
const c = watch({ aggregateTimeout: 20, ignored: /node_modules/ });
92+
93+
expect(b.watcherManager).toBe(a.watcherManager);
94+
expect(c.watcherManager).not.toBe(a.watcherManager);
95+
});
96+
97+
// The pool is keyed by value, so the entry must not stay attached to the
98+
// caller's object. Values are unique to this case: the map is module state.
99+
it("does not hand a later caller options mutated by an earlier one", () => {
100+
const mutated = { aggregateTimeout: 4321, ignored: ["**/mutated"] };
101+
watch(mutated);
102+
mutated.aggregateTimeout = 5000;
103+
mutated.ignored[0] = "**/other";
104+
105+
const later = watch({ aggregateTimeout: 4321, ignored: ["**/mutated"] });
106+
107+
expect(later.aggregateTimeout).toBe(4321);
108+
expect(later.options.ignored).toEqual(["**/mutated"]);
109+
});
110+
111+
it("leaves an unsupported ignored option to watchpack", () => {
112+
expect(() =>
113+
watch({
114+
aggregateTimeout: 20,
115+
ignored: /** @type {EXPECTED_ANY} */ (42)
116+
})
117+
).toThrow("Invalid option for 'ignored'");
118+
});
119+
});

0 commit comments

Comments
 (0)