Skip to content

Commit c78f937

Browse files
danbaodanbao
authored
fix(gateway): degrade config watcher to polling
Fixes #92851. When native filesystem watching exhausts its retry budget, the gateway config reloader now falls back to polling instead of disabling hot reload for the rest of the process. The watcher state tracks the effective Chokidar polling mode, including CHOKIDAR_USEPOLLING overrides, so forced polling avoids a redundant native phase and forced native mode reports an accurate native-mode disable. Verification: - node scripts/run-vitest.mjs src/gateway/config-reload.test.ts --maxWorkers=1 - .agents/skills/autoreview/scripts/autoreview --mode branch --base origin/main - Testbox-through-Crabbox tbx_01kv2xvbqkv4dmvvvsswzm75hz: OPENCLAW_CHECK_CHANGED_REMOTE_CHILD=1 OPENCLAW_CHANGED_LANES_RAW_SYNC=1 corepack pnpm check:changed - GitHub PR checks clean on c9762c5
1 parent d4a74b4 commit c78f937

2 files changed

Lines changed: 217 additions & 33 deletions

File tree

‎src/gateway/config-reload.test.ts‎

Lines changed: 175 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1643,37 +1643,183 @@ describe("startGatewayConfigReloader watcher error recovery", () => {
16431643
await reloader.stop();
16441644
});
16451645

1646-
it("disables hot-reload and logs at error level after the retry budget is exhausted", async () => {
1647-
const watchers = [
1648-
createWatcherMock(),
1649-
createWatcherMock(),
1650-
createWatcherMock(),
1651-
createWatcherMock(),
1652-
];
1653-
const { watchSpy, log, reloader } = startReloaderWithWatchers(watchers);
1654-
1655-
// Three errors consume the retry budget; the fourth error escalates.
1656-
watchers[0]?.emit("error");
1657-
await vi.advanceTimersByTimeAsync(500);
1658-
watchers[1]?.emit("error");
1659-
await vi.advanceTimersByTimeAsync(2000);
1660-
watchers[2]?.emit("error");
1661-
await vi.advanceTimersByTimeAsync(5000);
1662-
expect(watchSpy).toHaveBeenCalledTimes(4);
1663-
expect(reloader.hotReloadStatus()).toBe("active");
1646+
it("degrades to polling then disables after both native and polling retries are exhausted", async () => {
1647+
const originalVitest = process.env.VITEST;
1648+
const originalChokidarPolling = process.env.CHOKIDAR_USEPOLLING;
1649+
delete process.env.VITEST;
1650+
delete process.env.CHOKIDAR_USEPOLLING;
1651+
let reloader: { stop: () => Promise<void>; hotReloadStatus: () => string } | undefined;
1652+
try {
1653+
// Native phase: initial watcher + 3 re-creates = 4 watchers.
1654+
// Polling phase: 1 polling re-create + 3 re-creates = 4 watchers.
1655+
const watchers = Array.from({ length: 8 }, () => createWatcherMock());
1656+
const started = startReloaderWithWatchers(watchers);
1657+
const { watchSpy, log } = started;
1658+
reloader = started.reloader;
1659+
const watchOptions = (index: number) =>
1660+
watchSpy.mock.calls[index]?.[1] as { usePolling?: boolean } | undefined;
1661+
1662+
// --- Native retry phase (3 retries) ---
1663+
expect(watchOptions(0)?.usePolling).toBe(false);
1664+
watchers[0]?.emit("error");
1665+
await vi.advanceTimersByTimeAsync(500);
1666+
expect(watchOptions(1)?.usePolling).toBe(false);
1667+
watchers[1]?.emit("error");
1668+
await vi.advanceTimersByTimeAsync(2000);
1669+
expect(watchOptions(2)?.usePolling).toBe(false);
1670+
watchers[2]?.emit("error");
1671+
await vi.advanceTimersByTimeAsync(5000);
1672+
expect(watchSpy).toHaveBeenCalledTimes(4);
1673+
expect(watchOptions(3)?.usePolling).toBe(false);
1674+
expect(reloader.hotReloadStatus()).toBe("active");
1675+
1676+
// Fourth native error triggers degradation to polling mode (not disabled).
1677+
watchers[3]?.emit("error");
1678+
expect(reloader.hotReloadStatus()).toBe("active");
1679+
expect(log.warn).toHaveBeenCalledWith(
1680+
expect.stringContaining("degrading to polling mode"),
1681+
);
1682+
await vi.advanceTimersByTimeAsync(500);
1683+
expect(watchSpy).toHaveBeenCalledTimes(5);
1684+
expect(watchOptions(4)?.usePolling).toBe(true);
1685+
1686+
// --- Polling retry phase (3 retries) ---
1687+
watchers[4]?.emit("error");
1688+
await vi.advanceTimersByTimeAsync(500);
1689+
expect(watchOptions(5)?.usePolling).toBe(true);
1690+
watchers[5]?.emit("error");
1691+
await vi.advanceTimersByTimeAsync(2000);
1692+
expect(watchOptions(6)?.usePolling).toBe(true);
1693+
watchers[6]?.emit("error");
1694+
await vi.advanceTimersByTimeAsync(5000);
1695+
expect(watchSpy).toHaveBeenCalledTimes(8);
1696+
expect(watchOptions(7)?.usePolling).toBe(true);
1697+
expect(reloader.hotReloadStatus()).toBe("active");
1698+
1699+
// Eighth error in polling mode finally disables hot-reload.
1700+
watchers[7]?.emit("error");
1701+
expect(reloader.hotReloadStatus()).toBe("disabled");
1702+
expect(log.error).toHaveBeenCalledWith(
1703+
expect.stringContaining(
1704+
"config hot-reload disabled: watcher failed after 3 re-create attempts in polling mode",
1705+
),
1706+
);
1707+
// No further watcher is created once disabled.
1708+
await vi.advanceTimersByTimeAsync(10000);
1709+
expect(watchSpy).toHaveBeenCalledTimes(8);
1710+
} finally {
1711+
if (originalVitest === undefined) {
1712+
delete process.env.VITEST;
1713+
} else {
1714+
process.env.VITEST = originalVitest;
1715+
}
1716+
if (originalChokidarPolling === undefined) {
1717+
delete process.env.CHOKIDAR_USEPOLLING;
1718+
} else {
1719+
process.env.CHOKIDAR_USEPOLLING = originalChokidarPolling;
1720+
}
1721+
await reloader?.stop();
1722+
}
1723+
});
16641724

1665-
watchers[3]?.emit("error");
1666-
expect(reloader.hotReloadStatus()).toBe("disabled");
1667-
expect(log.error).toHaveBeenCalledWith(
1668-
expect.stringContaining(
1669-
"config hot-reload disabled: watcher failed after 3 re-create attempts",
1670-
),
1671-
);
1672-
// No further watcher is created once disabled.
1673-
await vi.advanceTimersByTimeAsync(10000);
1674-
expect(watchSpy).toHaveBeenCalledTimes(4);
1725+
it("does not run a redundant native phase when chokidar polling is forced on", async () => {
1726+
const originalVitest = process.env.VITEST;
1727+
const originalChokidarPolling = process.env.CHOKIDAR_USEPOLLING;
1728+
delete process.env.VITEST;
1729+
process.env.CHOKIDAR_USEPOLLING = "1";
1730+
let reloader: { stop: () => Promise<void>; hotReloadStatus: () => string } | undefined;
1731+
try {
1732+
const watchers = Array.from({ length: 4 }, () => createWatcherMock());
1733+
const started = startReloaderWithWatchers(watchers);
1734+
const { watchSpy, log } = started;
1735+
reloader = started.reloader;
1736+
const watchOptions = (index: number) =>
1737+
watchSpy.mock.calls[index]?.[1] as { usePolling?: boolean } | undefined;
1738+
1739+
expect(watchOptions(0)?.usePolling).toBe(true);
1740+
watchers[0]?.emit("error");
1741+
await vi.advanceTimersByTimeAsync(500);
1742+
expect(watchOptions(1)?.usePolling).toBe(true);
1743+
watchers[1]?.emit("error");
1744+
await vi.advanceTimersByTimeAsync(2000);
1745+
expect(watchOptions(2)?.usePolling).toBe(true);
1746+
watchers[2]?.emit("error");
1747+
await vi.advanceTimersByTimeAsync(5000);
1748+
expect(watchOptions(3)?.usePolling).toBe(true);
1749+
1750+
watchers[3]?.emit("error");
1751+
expect(reloader.hotReloadStatus()).toBe("disabled");
1752+
expect(log.warn).not.toHaveBeenCalledWith(
1753+
expect.stringContaining("degrading to polling mode"),
1754+
);
1755+
expect(log.error).toHaveBeenCalledWith(
1756+
expect.stringContaining(
1757+
"config hot-reload disabled: watcher failed after 3 re-create attempts in polling mode",
1758+
),
1759+
);
1760+
} finally {
1761+
if (originalVitest === undefined) {
1762+
delete process.env.VITEST;
1763+
} else {
1764+
process.env.VITEST = originalVitest;
1765+
}
1766+
if (originalChokidarPolling === undefined) {
1767+
delete process.env.CHOKIDAR_USEPOLLING;
1768+
} else {
1769+
process.env.CHOKIDAR_USEPOLLING = originalChokidarPolling;
1770+
}
1771+
await reloader?.stop();
1772+
}
1773+
});
16751774

1676-
await reloader.stop();
1775+
it("does not report polling fallback when chokidar polling is forced off", async () => {
1776+
const originalVitest = process.env.VITEST;
1777+
const originalChokidarPolling = process.env.CHOKIDAR_USEPOLLING;
1778+
delete process.env.VITEST;
1779+
process.env.CHOKIDAR_USEPOLLING = "0";
1780+
let reloader: { stop: () => Promise<void>; hotReloadStatus: () => string } | undefined;
1781+
try {
1782+
const watchers = Array.from({ length: 4 }, () => createWatcherMock());
1783+
const started = startReloaderWithWatchers(watchers);
1784+
const { watchSpy, log } = started;
1785+
reloader = started.reloader;
1786+
const watchOptions = (index: number) =>
1787+
watchSpy.mock.calls[index]?.[1] as { usePolling?: boolean } | undefined;
1788+
1789+
expect(watchOptions(0)?.usePolling).toBe(false);
1790+
watchers[0]?.emit("error");
1791+
await vi.advanceTimersByTimeAsync(500);
1792+
expect(watchOptions(1)?.usePolling).toBe(false);
1793+
watchers[1]?.emit("error");
1794+
await vi.advanceTimersByTimeAsync(2000);
1795+
expect(watchOptions(2)?.usePolling).toBe(false);
1796+
watchers[2]?.emit("error");
1797+
await vi.advanceTimersByTimeAsync(5000);
1798+
expect(watchOptions(3)?.usePolling).toBe(false);
1799+
1800+
watchers[3]?.emit("error");
1801+
expect(reloader.hotReloadStatus()).toBe("disabled");
1802+
expect(log.warn).not.toHaveBeenCalledWith(
1803+
expect.stringContaining("degrading to polling mode"),
1804+
);
1805+
expect(log.error).toHaveBeenCalledWith(
1806+
expect.stringContaining(
1807+
"config hot-reload disabled: watcher failed after 3 re-create attempts in native mode",
1808+
),
1809+
);
1810+
} finally {
1811+
if (originalVitest === undefined) {
1812+
delete process.env.VITEST;
1813+
} else {
1814+
process.env.VITEST = originalVitest;
1815+
}
1816+
if (originalChokidarPolling === undefined) {
1817+
delete process.env.CHOKIDAR_USEPOLLING;
1818+
} else {
1819+
process.env.CHOKIDAR_USEPOLLING = originalChokidarPolling;
1820+
}
1821+
await reloader?.stop();
1822+
}
16771823
});
16781824
});
16791825

‎src/gateway/config-reload.ts‎

Lines changed: 42 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -35,11 +35,27 @@ const MISSING_CONFIG_MAX_RETRIES = 2;
3535

3636
// Watcher 'error' events (for example EMFILE/ENOSPC inotify exhaustion) close
3737
// the chokidar watcher. Re-create it with bounded backoff so a transient fault
38-
// does not permanently kill config hot-reload, but escalate to error + a
39-
// persistent disabled status once the retry budget is exhausted.
38+
// does not permanently kill config hot-reload. If all native retries are
39+
// exhausted (typical when the host has insufficient inotify watches), fall
40+
// back to polling mode before giving up entirely.
4041
const WATCHER_RECREATE_MAX_RETRIES = 3;
4142
const WATCHER_RECREATE_BACKOFF_MS = [500, 2000, 5000] as const;
4243

44+
function resolveChokidarUsePolling(degradedToPolling: boolean): boolean {
45+
const envPoll = process.env.CHOKIDAR_USEPOLLING;
46+
if (envPoll !== undefined) {
47+
const envLower = envPoll.toLowerCase();
48+
if (envLower === "false" || envLower === "0") {
49+
return false;
50+
}
51+
if (envLower === "true" || envLower === "1") {
52+
return true;
53+
}
54+
return Boolean(envLower);
55+
}
56+
return Boolean(process.env.VITEST) || degradedToPolling;
57+
}
58+
4359
/**
4460
* Paths under `skills.*` always change the snapshot that sessions cache in
4561
* sessions.json. Any prefix match here (for example `skills.allowBundled`,
@@ -407,15 +423,18 @@ export function startGatewayConfigReloader(opts: {
407423
let watcherRecreateRetries = 0;
408424
let watcherRecreateTimer: ReturnType<typeof setTimeout> | null = null;
409425
let hotReloadStatus: GatewayHotReloadStatus = "active";
426+
let degradedToPolling = false;
427+
let watcherUsesPolling = false;
410428

411429
const createWatcher = () => {
412430
if (stopped) {
413431
return;
414432
}
433+
const usePolling = resolveChokidarUsePolling(degradedToPolling);
415434
const next = chokidar.watch(opts.watchPath, {
416435
ignoreInitial: true,
417436
awaitWriteFinish: { stabilityThreshold: 200, pollInterval: 50 },
418-
usePolling: Boolean(process.env.VITEST),
437+
usePolling,
419438
});
420439
next.on("add", scheduleFromWatcher);
421440
next.on("change", scheduleFromWatcher);
@@ -424,6 +443,7 @@ export function startGatewayConfigReloader(opts: {
424443
handleWatcherError(next, err);
425444
});
426445
watcher = next;
446+
watcherUsesPolling = usePolling;
427447
hotReloadStatus = "active";
428448
};
429449

@@ -432,12 +452,30 @@ export function startGatewayConfigReloader(opts: {
432452
if (stopped || source !== watcher) {
433453
return;
434454
}
455+
const failedWatcherUsedPolling = watcherUsesPolling;
435456
watcher = null;
457+
watcherUsesPolling = false;
436458
void source?.close().catch(() => {});
437459
if (watcherRecreateRetries >= WATCHER_RECREATE_MAX_RETRIES) {
460+
// All native (inotify/kqueue) retries exhausted — fall back to polling
461+
// mode so config hot-reload survives on hosts where inotify resources
462+
// are constrained (e.g. low fs.inotify.max_user_watches).
463+
if (!failedWatcherUsedPolling && resolveChokidarUsePolling(true)) {
464+
degradedToPolling = true;
465+
watcherRecreateRetries = 0;
466+
opts.log.warn(
467+
`config watcher native retries exhausted; degrading to polling mode: ${String(err)}`,
468+
);
469+
watcherRecreateTimer = setTimeout(() => {
470+
watcherRecreateTimer = null;
471+
createWatcher();
472+
}, WATCHER_RECREATE_BACKOFF_MS[0] ?? 500);
473+
return;
474+
}
475+
const mode = failedWatcherUsedPolling ? "polling mode" : "native mode";
438476
hotReloadStatus = "disabled";
439477
opts.log.error(
440-
`config hot-reload disabled: watcher failed after ${WATCHER_RECREATE_MAX_RETRIES} re-create attempts: ${String(err)}`,
478+
`config hot-reload disabled: watcher failed after ${WATCHER_RECREATE_MAX_RETRIES} re-create attempts in ${mode}: ${String(err)}`,
441479
);
442480
return;
443481
}

0 commit comments

Comments
 (0)