Skip to content

Commit 2ef4ad3

Browse files
Merge pull request #359 from webpack/fix/watcher-limit-errors
fix: poll directories the system has no watcher left for, instead of reporting their files as removed
2 parents 0418015 + e3cb35e commit 2ef4ad3

4 files changed

Lines changed: 126 additions & 0 deletions

File tree

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"watchpack": patch
3+
---
4+
5+
When the system runs out of file watchers (`ENOSPC`/`EMFILE`, e.g. `fs.inotify.max_user_watches` on Linux), poll the directories it cannot watch instead of reporting their files as removed and retrying in a loop, and log the error once with a hint (fixes #46).

‎lib/DirectoryWatcher.js‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,12 @@ const EXISTANCE_ONLY_TIME_ENTRY = Object.freeze({});
2525

2626
let FS_ACCURACY = 2000;
2727

28+
// the OS has no watcher left for the directory, which is still there
29+
const WATCHER_LIMIT_ERRORS = new Set(["ENOSPC", "EMFILE"]);
30+
// the polling interval of `poll: true`
31+
const WATCHER_LIMIT_POLL_INTERVAL = 5007;
32+
let watcherLimitReported = false;
33+
2834
const IS_OSX = require("os").platform() === "darwin";
2935
const IS_WIN = require("os").platform() === "win32";
3036

@@ -719,6 +725,16 @@ class DirectoryWatcher extends EventEmitter {
719725
onWatcherError(err) {
720726
if (this.closed) return;
721727
if (err) {
728+
if (
729+
WATCHER_LIMIT_ERRORS.has(
730+
/** @type {string} */ (
731+
/** @type {NodeJS.ErrnoException} */ (err).code
732+
),
733+
)
734+
) {
735+
this.pollAfterWatcherLimit(err);
736+
return;
737+
}
722738
if (
723739
/** @type {NodeJS.ErrnoException} */
724740
(err).code !== "EPERM" &&
@@ -732,6 +748,25 @@ class DirectoryWatcher extends EventEmitter {
732748
}
733749
}
734750

751+
/**
752+
* Polls the directory, which still exists, when the OS cannot watch it.
753+
* @param {unknown} err watcher limit error
754+
*/
755+
pollAfterWatcherLimit(err) {
756+
if (!watcherLimitReported) {
757+
watcherLimitReported = true;
758+
// eslint-disable-next-line no-console
759+
console.error(
760+
`Watchpack Error (watcher): ${err}\nWatchpack polls the directories it cannot watch. Raise the limit of the system (e.g. fs.inotify.max_user_watches on Linux) or set WATCHPACK_POLLING.`,
761+
);
762+
}
763+
if (this.watcher) this.watcher.close();
764+
this.polledWatching = WATCHER_LIMIT_POLL_INTERVAL;
765+
this.createWatcher();
766+
// a finishing initial scan starts the polling by itself
767+
if (!this.initialScan) this.doScan(false);
768+
}
769+
735770
/**
736771
* @param {Error | NodeJS.ErrnoException=} err error
737772
*/

‎test/WatcherLimit.test.js‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
"use strict";
2+
3+
const fs = require("fs");
4+
const path = require("path");
5+
const Watchpack = require("../lib");
6+
const TestHelper = require("./helpers/TestHelper");
7+
8+
const fixtures = path.join(__dirname, "fixtures");
9+
const testHelper = new TestHelper(fixtures);
10+
11+
// eslint-disable-next-line jest/no-confusing-set-timeout
12+
jest.setTimeout(20000);
13+
14+
describe("Watcher limit of the system (#46)", () => {
15+
beforeEach((done) => {
16+
testHelper.before(done);
17+
});
18+
19+
/** @type {InstanceType<typeof Watchpack>[]} */
20+
let watchers = [];
21+
22+
afterEach((done) => {
23+
// closed here, not in the test, so a failure cannot leave one running
24+
for (const w of watchers) w.close();
25+
watchers = [];
26+
jest.restoreAllMocks();
27+
testHelper.after(done);
28+
});
29+
30+
it("should poll a directory it cannot watch instead of reporting its files as removed", (done) => {
31+
const dir = path.join(fixtures, "dir");
32+
const file = path.join(dir, "a");
33+
testHelper.dir("dir");
34+
testHelper.file(path.join("dir", "a"));
35+
const { watch } = fs;
36+
let watchCalls = 0;
37+
jest.spyOn(fs, "watch").mockImplementation((target, ...args) => {
38+
if (target === dir) {
39+
watchCalls++;
40+
throw Object.assign(
41+
new Error(
42+
`ENOSPC: System limit for number of file watchers reached, watch '${dir}'`,
43+
),
44+
{ code: "ENOSPC" },
45+
);
46+
}
47+
return /** @type {(...a: unknown[]) => fs.FSWatcher} */ (watch)(
48+
target,
49+
...args,
50+
);
51+
});
52+
const errors = jest.spyOn(console, "error").mockImplementation(() => {});
53+
testHelper.tick(1000, () => {
54+
const w = new Watchpack({ aggregateTimeout: 100 });
55+
watchers.push(w);
56+
/** @type {string[]} */
57+
const removed = [];
58+
w.on("remove", (item) => {
59+
removed.push(item);
60+
});
61+
let edited = false;
62+
w.on("change", (item) => {
63+
// coarse timestamps can report the file in the initial scan
64+
if (!edited) return;
65+
expect(item).toBe(file);
66+
expect(removed).toEqual([]);
67+
// tried once, not again and again
68+
expect(watchCalls).toBeLessThanOrEqual(1);
69+
expect(
70+
errors.mock.calls.filter((call) => /ENOSPC/.test(String(call[0]))),
71+
).toHaveLength(watchCalls);
72+
done();
73+
});
74+
w.watch({ files: [file], startTime: Date.now() });
75+
testHelper.tick(1000, () => {
76+
edited = true;
77+
testHelper.file(path.join("dir", "a"));
78+
});
79+
});
80+
});
81+
});

‎types/DirectoryWatcher.d.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -151,6 +151,11 @@ declare class DirectoryWatcher extends EventEmitter<{
151151
* @param {unknown=} err error
152152
*/
153153
onWatcherError(err?: unknown | undefined): void;
154+
/**
155+
* Polls the directory, which still exists, when the OS cannot watch it.
156+
* @param {unknown} err watcher limit error
157+
*/
158+
pollAfterWatcherLimit(err: unknown): void;
154159
/**
155160
* @param {Error | NodeJS.ErrnoException=} err error
156161
*/

0 commit comments

Comments
 (0)