Skip to content

Commit 2405bbc

Browse files
RomneyDaRomneyDa
authored
fix(memory): warn on gateway watcher FD risk (#89185)
* fix(memory): default gateway memory watch off * fix(memory): warn on gateway watcher fd risk * fix(config): avoid warning helper narrowing * fix(config): remove redundant warning boolean cast * docs(memory): clarify watcher default wording * docs(memory): simplify watcher warning copy * fix(config): scope watcher warning to local gateway
1 parent 4031905 commit 2405bbc

10 files changed

Lines changed: 262 additions & 7 deletions

File tree

‎docs/concepts/memory-builtin.md‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,10 @@ OpenClaw indexes `MEMORY.md` and `memory/*.md` into chunks (~400 tokens with
8585
- **Storage maintenance:** SQLite WAL sidecars are bounded with periodic and
8686
shutdown checkpoints.
8787
- **File watching:** changes to memory files trigger a debounced reindex (1.5s).
88+
File watching is enabled by default, including for gateways, so memory edits
89+
become searchable without a manual reindex. Large memory trees, `extraPaths`,
90+
or QMD collections can use many file descriptors in long-lived gateways; set
91+
`sync.watch: false` for affected agents if that becomes a problem.
8892
- **Auto-reindex:** when the embedding provider, model, or chunking config
8993
changes, the entire index is rebuilt automatically.
9094
- **Reindex on demand:** `openclaw memory index --force`
@@ -125,8 +129,8 @@ openclaw memory index --force --agent main
125129
Both standalone CLI commands and the Gateway use the same `local` provider id.
126130
Set `memorySearch.provider: "local"` when you want local embeddings.
127131

128-
**Stale results?** Run `openclaw memory index --force` to rebuild. The watcher
129-
may miss changes in rare edge cases.
132+
**Stale results?** Run `openclaw memory index --force` to rebuild. Use this when
133+
file watching is disabled or misses a change.
130134

131135
**sqlite-vec not loading?** OpenClaw falls back to in-process cosine similarity
132136
automatically. `openclaw memory status --deep` reports the local vector store

‎docs/reference/memory-config.md‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -527,7 +527,9 @@ QMD model overrides stay on the QMD side, not OpenClaw config. If you need to ov
527527
</Accordion>
528528
</AccordionGroup>
529529

530-
QMD boot refreshes use a one-shot subprocess path during gateway startup. The long-lived QMD manager still owns the regular file watcher and interval timers when memory search is opened for interactive use.
530+
QMD boot refreshes use a one-shot subprocess path during gateway startup. The long-lived QMD manager owns the regular file watcher and interval timers when memory search is opened for interactive use.
531+
532+
Local Gateway configs can warn when memory file watching may keep too many files open. If you see open-file or watcher errors, set `sync.watch: false` for the affected agents and use manual indexing or `sync.intervalMinutes` to refresh memory.
531533

532534
### Full QMD example
533535

‎extensions/memory-core/src/memory/qmd-manager.test.ts‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -569,6 +569,41 @@ describe("QmdMemoryManager", () => {
569569
await manager.close();
570570
});
571571

572+
it("logs qmd watcher errors without throwing", async () => {
573+
cfg = {
574+
agents: {
575+
defaults: {
576+
workspace: workspaceDir,
577+
memorySearch: {
578+
provider: "openai",
579+
model: "mock-embed",
580+
store: { path: path.join(workspaceDir, "index.sqlite"), vector: { enabled: false } },
581+
sync: { watch: true, watchDebounceMs: 25, onSessionStart: false, onSearch: false },
582+
},
583+
},
584+
list: [{ id: agentId, default: true, workspace: workspaceDir }],
585+
},
586+
memory: {
587+
backend: "qmd",
588+
qmd: {
589+
includeDefaultMemory: false,
590+
update: { interval: "0s", debounceMs: 0, onBoot: false },
591+
paths: [{ path: workspaceDir, pattern: "**/*.md", name: "workspace" }],
592+
},
593+
},
594+
} as OpenClawConfig;
595+
596+
const { manager } = await createManager({ mode: "full" });
597+
const watcher = watchMock.mock.results[0]?.value as {
598+
emit: (event: string, ...args: unknown[]) => boolean;
599+
};
600+
601+
expect(watcher.emit("error", new Error("watcher error: ENOSPC"))).toBe(true);
602+
expect(logWarnMock).toHaveBeenCalledWith("qmd watcher error: watcher error: ENOSPC");
603+
604+
await manager.close();
605+
});
606+
572607
it("delays qmd watch sync until changed file stats settle", async () => {
573608
vi.useFakeTimers();
574609
cfg = {

‎extensions/memory-core/src/memory/qmd-manager.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1616,6 +1616,10 @@ export class QmdMemoryManager implements MemorySearchManager {
16161616
this.watcher.on("add", markDirty);
16171617
this.watcher.on("change", markDirty);
16181618
this.watcher.on("unlink", markDirty);
1619+
this.watcher.on("error", (err) => {
1620+
const message = err instanceof Error ? err.message : String(err);
1621+
log.warn(`qmd watcher error: ${message}`);
1622+
});
16191623
this.watcher.once("ready", () => {
16201624
log.info(
16211625
`qmd watcher ready for agent "${this.agentId}" paths=${watchPathList.length} durationMs=${Date.now() - startTime}`,

‎src/agents/memory-search.test.ts‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -291,6 +291,22 @@ describe("memory search config", () => {
291291
expect(resolveMemorySearchSyncConfig(cfg, "main")?.embeddingBatchTimeoutSeconds).toBe(600);
292292
});
293293

294+
it("keeps memory watching enabled by default in gateway mode", () => {
295+
const cfg = asConfig({
296+
gateway: { mode: "local" },
297+
agents: {
298+
defaults: {
299+
memorySearch: {
300+
provider: "openai",
301+
},
302+
},
303+
},
304+
});
305+
306+
expect(resolveMemorySearchConfig(cfg, "main")?.sync.watch).toBe(true);
307+
expect(resolveMemorySearchSyncConfig(cfg, "main")?.watch).toBe(true);
308+
});
309+
294310
it("merges defaults and overrides", () => {
295311
const cfg = asConfig({
296312
agents: {

‎src/agents/memory-search.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,11 @@ export type ResolvedMemorySearchConfig = {
111111
};
112112

113113
export type ResolvedMemorySearchSyncConfig = ResolvedMemorySearchConfig["sync"];
114+
export type MemorySearchResolvePurpose = "default" | "status" | "cli";
115+
export type MemorySearchResolveOptions = {
116+
/** @deprecated No-op; kept for resolver call-site compatibility. */
117+
purpose?: MemorySearchResolvePurpose;
118+
};
114119

115120
const DEFAULT_CHUNK_TOKENS = 400;
116121
const DEFAULT_CHUNK_OVERLAP = 80;
@@ -468,6 +473,7 @@ function resolveSyncConfig(
468473
export function resolveMemorySearchConfig(
469474
cfg: OpenClawConfig,
470475
agentId: string,
476+
_options?: MemorySearchResolveOptions,
471477
): ResolvedMemorySearchConfig | null {
472478
const defaults = cfg.agents?.defaults?.memorySearch;
473479
const overrides = resolveAgentConfig(cfg, agentId)?.memorySearch;
@@ -500,6 +506,7 @@ export function resolveMemorySearchConfig(
500506
export function resolveMemorySearchSyncConfig(
501507
cfg: OpenClawConfig,
502508
agentId: string,
509+
_options?: MemorySearchResolveOptions,
503510
): ResolvedMemorySearchSyncConfig | null {
504511
const defaults = cfg.agents?.defaults?.memorySearch;
505512
const overrides = resolveAgentConfig(cfg, agentId)?.memorySearch;

‎src/config/schema.help.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1331,7 +1331,7 @@ export const FIELD_HELP: Record<string, string> = {
13311331
"agents.defaults.memorySearch.sync.onSearch":
13321332
"Uses lazy sync by scheduling reindex on search after content changes are detected. Keep enabled for lower idle overhead, or disable if you require pre-synced indexes before any query.",
13331333
"agents.defaults.memorySearch.sync.watch":
1334-
"Watches memory files and schedules index updates from file-change events (chokidar). Enable for near-real-time freshness; disable on very large workspaces if watch churn is too noisy.",
1334+
"Watches memory files and schedules index updates from file-change events. Default: true. Disable for large memory trees, extraPaths, QMD collections, or multi-agent gateways if watcher file-descriptor usage becomes a problem.",
13351335
"agents.defaults.memorySearch.sync.watchDebounceMs":
13361336
"Debounce window in milliseconds for coalescing rapid file-watch events before reindex runs. Increase to reduce churn on frequently-written files, or lower for faster freshness.",
13371337
"agents.defaults.memorySearch.sync.embeddingBatchTimeoutSeconds":

‎src/config/types.tools.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -523,6 +523,7 @@ export type MemorySearchConfig = {
523523
sync?: {
524524
onSessionStart?: boolean;
525525
onSearch?: boolean;
526+
/** Watch memory files for reindexing (default: true). */
526527
watch?: boolean;
527528
watchDebounceMs?: number;
528529
intervalMinutes?: number;

‎src/config/validation.policy.test.ts‎

Lines changed: 103 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import { describe, expect, it, vi } from "vitest";
2-
import { validateConfigObjectRaw } from "./validation.js";
2+
import { validateConfigObjectRaw, validateConfigObjectWithPlugins } from "./validation.js";
33

44
vi.mock("../channels/plugins/legacy-config.js", () => ({
55
collectChannelLegacyConfigRules: () => [],
@@ -41,6 +41,108 @@ vi.mock("../secrets/unsupported-surface-policy.js", async () => {
4141
};
4242
});
4343

44+
describe("gateway memory watch config warnings", () => {
45+
it("warns when gateway memory watching stays enabled on configured memory surfaces", () => {
46+
const result = validateConfigObjectWithPlugins(
47+
{
48+
gateway: { mode: "local" },
49+
agents: {
50+
defaults: {
51+
memorySearch: {
52+
extraPaths: ["/srv/shared-notes"],
53+
},
54+
},
55+
},
56+
},
57+
{ pluginValidation: "skip" },
58+
);
59+
60+
expect(result.ok).toBe(true);
61+
expect(result.warnings).toContainEqual(
62+
expect.objectContaining({
63+
path: "agents.defaults.memorySearch.sync.watch",
64+
message: expect.stringContaining("too many files open"),
65+
}),
66+
);
67+
});
68+
69+
it("does not warn when gateway memory watching is disabled explicitly", () => {
70+
const result = validateConfigObjectWithPlugins(
71+
{
72+
gateway: { mode: "local" },
73+
agents: {
74+
defaults: {
75+
memorySearch: {
76+
extraPaths: ["/srv/shared-notes"],
77+
sync: { watch: false },
78+
},
79+
},
80+
},
81+
},
82+
{ pluginValidation: "skip" },
83+
);
84+
85+
expect(result.ok).toBe(true);
86+
expect(result.warnings).not.toContainEqual(
87+
expect.objectContaining({
88+
path: "agents.defaults.memorySearch.sync.watch",
89+
}),
90+
);
91+
});
92+
93+
it("does not warn for remote client configs", () => {
94+
const result = validateConfigObjectWithPlugins(
95+
{
96+
gateway: { mode: "remote", remote: { url: "wss://gateway.example/ws" } },
97+
agents: {
98+
defaults: {
99+
memorySearch: {
100+
extraPaths: ["/srv/shared-notes"],
101+
sync: { watch: true },
102+
},
103+
},
104+
},
105+
},
106+
{ pluginValidation: "skip" },
107+
);
108+
109+
expect(result.ok).toBe(true);
110+
expect(result.warnings).not.toContainEqual(
111+
expect.objectContaining({
112+
path: "agents.defaults.memorySearch.sync.watch",
113+
}),
114+
);
115+
});
116+
117+
it("warns for explicit per-agent watcher overrides in multi-agent gateways", () => {
118+
const result = validateConfigObjectWithPlugins(
119+
{
120+
gateway: { mode: "local" },
121+
agents: {
122+
defaults: {
123+
memorySearch: {
124+
sync: { watch: false },
125+
},
126+
},
127+
list: [
128+
{ id: "main", memorySearch: { sync: { watch: false } } },
129+
{ id: "ops", memorySearch: { sync: { watch: true } } },
130+
],
131+
},
132+
},
133+
{ pluginValidation: "skip" },
134+
);
135+
136+
expect(result.ok).toBe(true);
137+
expect(result.warnings).toContainEqual(
138+
expect.objectContaining({
139+
path: "agents.list.1.memorySearch.sync.watch",
140+
message: expect.stringContaining("many agents"),
141+
}),
142+
);
143+
});
144+
});
145+
44146
function requireIssue<T extends { path: string }>(issues: T[], path: string): T {
45147
const issue = issues.find((entry) => entry.path === path);
46148
if (!issue) {

‎src/config/validation.ts‎

Lines changed: 86 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ import { shouldSuppressMissingCodexPluginDiagnostics } from "./codex-plugin-diag
4646
import { materializeRuntimeConfig } from "./materialize.js";
4747
import type { OpenClawConfig, ConfigValidationIssue } from "./types.js";
4848
import { coerceSecretRef } from "./types.secrets.js";
49+
import type { MemorySearchConfig } from "./types.tools.js";
4950
import { isBuiltInModelProviderOverlayId } from "./zod-schema.core.js";
5051
import { OpenClawSchema } from "./zod-schema.js";
5152

@@ -871,6 +872,88 @@ function validateGatewayTailscaleAuth(config: OpenClawConfig): ConfigValidationI
871872
];
872873
}
873874

875+
function isLocalGatewayMode(config: OpenClawConfig): boolean {
876+
return config.gateway?.mode === "local";
877+
}
878+
879+
function isMemorySearchEnabled(
880+
defaults: MemorySearchConfig | undefined,
881+
override: MemorySearchConfig | undefined,
882+
): boolean {
883+
return override?.enabled ?? defaults?.enabled ?? true;
884+
}
885+
886+
function isMemoryWatchEnabled(
887+
defaults: MemorySearchConfig | undefined,
888+
override: MemorySearchConfig | undefined,
889+
): boolean {
890+
return override?.sync?.watch ?? defaults?.sync?.watch ?? true;
891+
}
892+
893+
function hasConfiguredMemoryWatchFdPressureSurface(
894+
config: OpenClawConfig,
895+
defaults: MemorySearchConfig | undefined,
896+
override: MemorySearchConfig | undefined,
897+
): boolean {
898+
const hasMemorySearchConfig = Boolean(defaults || override);
899+
const hasMultipleGatewayAgents = (config.agents?.list?.length ?? 0) > 1;
900+
const hasQmdBackend = config.memory?.backend === "qmd";
901+
const hasQmdPaths = Boolean(config.memory?.qmd?.paths?.length);
902+
const hasExtraPaths = Boolean(defaults?.extraPaths?.length || override?.extraPaths?.length);
903+
const hasExtraQmdCollections = Boolean(
904+
defaults?.qmd?.extraCollections?.length || override?.qmd?.extraCollections?.length,
905+
);
906+
const hasSessionMemory = Boolean(
907+
defaults?.experimental?.sessionMemory || override?.experimental?.sessionMemory,
908+
);
909+
return (
910+
hasMemorySearchConfig ||
911+
hasMultipleGatewayAgents ||
912+
hasQmdBackend ||
913+
hasQmdPaths ||
914+
hasExtraPaths ||
915+
hasExtraQmdCollections ||
916+
hasSessionMemory
917+
);
918+
}
919+
920+
function memoryWatchFdPressureWarningMessage(): string {
921+
return "Memory file watching is on for this Gateway. This keeps memory search up to date, but large memory folders, extraPaths, QMD collections, session memory, or many agents can make the Gateway keep too many files open. If you see open-file or watcher errors, set memorySearch.sync.watch: false for the affected default or agent, then use manual indexing or sync.intervalMinutes to refresh memory.";
922+
}
923+
924+
function collectGatewayMemoryWatchWarnings(config: OpenClawConfig): ConfigValidationIssue[] {
925+
if (!isLocalGatewayMode(config)) {
926+
return [];
927+
}
928+
const defaults = config.agents?.defaults?.memorySearch;
929+
const warnings: ConfigValidationIssue[] = [];
930+
if (
931+
isMemorySearchEnabled(defaults, undefined) &&
932+
isMemoryWatchEnabled(defaults, undefined) &&
933+
hasConfiguredMemoryWatchFdPressureSurface(config, defaults, undefined)
934+
) {
935+
warnings.push({
936+
path: "agents.defaults.memorySearch.sync.watch",
937+
message: memoryWatchFdPressureWarningMessage(),
938+
});
939+
}
940+
for (const [index, agent] of (config.agents?.list ?? []).entries()) {
941+
const override = agent.memorySearch;
942+
if (
943+
override &&
944+
isMemorySearchEnabled(defaults, override) &&
945+
isMemoryWatchEnabled(defaults, override) &&
946+
hasConfiguredMemoryWatchFdPressureSurface(config, defaults, override)
947+
) {
948+
warnings.push({
949+
path: `agents.list.${index}.memorySearch.sync.watch`,
950+
message: memoryWatchFdPressureWarningMessage(),
951+
});
952+
}
953+
}
954+
return warnings;
955+
}
956+
874957
/**
875958
* Validates config without applying runtime defaults.
876959
* Use this when you need the raw validated config (e.g., for writing back to file).
@@ -1039,16 +1122,17 @@ function validateConfigObjectWithPluginsBase(
10391122
manifestRegistry: registryInfo?.registry,
10401123
})
10411124
: base.config;
1125+
const configWarnings = collectGatewayMemoryWatchWarnings(base.config);
10421126
if (opts.pluginValidation === "skip") {
10431127
return {
10441128
ok: true,
10451129
config,
1046-
warnings: [],
1130+
warnings: configWarnings,
10471131
};
10481132
}
10491133

10501134
const issues: ConfigValidationIssue[] = [];
1051-
const warnings: ConfigValidationIssue[] = [];
1135+
const warnings: ConfigValidationIssue[] = [...configWarnings];
10521136
const hasExplicitPluginsConfig = isRecord(raw) && Object.hasOwn(raw, "plugins");
10531137
const explicitPluginReferences = collectExplicitPluginReferences(raw);
10541138

0 commit comments

Comments
 (0)