Skip to content

Commit 543f2b9

Browse files
author
OpenClaw Assistant
committed
Fix memory_search tool timeout handling
Classify memory_search tool deadline failures separately from embedding provider errors, add a configurable memory-core tool timeout, and return partial corpus=all results when a supplemental corpus stalls after primary memory results are available.
1 parent 561b293 commit 543f2b9

6 files changed

Lines changed: 231 additions & 18 deletions

File tree

‎docs/concepts/memory-search.md‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -159,6 +159,14 @@ inline batch timeout by default. If the host is simply slow, set
159159
`agents.defaults.memorySearch.sync.embeddingBatchTimeoutSeconds` and rerun
160160
`openclaw memory index --force`.
161161

162+
**`memory_search` itself times out?** The tool has a separate hard deadline so a
163+
slow Gateway or large `memory+sessions` index cannot block the agent turn
164+
indefinitely. Timeout results are reported as tool-deadline failures rather than
165+
embedding-provider failures. For slow hosts, set
166+
`plugins.entries.memory-core.config.tools.memorySearch.timeoutMs` (milliseconds,
167+
1000-120000). For broad `corpus="all"` searches, OpenClaw returns partial
168+
results when one corpus finishes before another stalls.
169+
162170
**CJK text not found?** Rebuild the FTS index with
163171
`openclaw memory index --force`.
164172

‎extensions/memory-core/openclaw.plugin.json‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,24 @@
3030
"type": "object",
3131
"additionalProperties": false,
3232
"properties": {
33+
"tools": {
34+
"type": "object",
35+
"additionalProperties": false,
36+
"properties": {
37+
"memorySearch": {
38+
"type": "object",
39+
"additionalProperties": false,
40+
"properties": {
41+
"timeoutMs": {
42+
"type": "integer",
43+
"minimum": 1000,
44+
"maximum": 120000,
45+
"description": "Hard deadline for the memory_search tool in milliseconds. Increase on slow hosts with large memory/session indexes."
46+
}
47+
}
48+
}
49+
}
50+
},
3351
"dreaming": {
3452
"type": "object",
3553
"additionalProperties": false,

‎extensions/memory-core/src/tools.citations.test.ts‎

Lines changed: 25 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -452,7 +452,7 @@ describe("memory tools", () => {
452452
expect(getMemorySearchManagerMockCalls()).toBe(1);
453453
});
454454

455-
it("does not cooldown primary memory when a corpus=all wiki supplement stalls", async () => {
455+
it("returns partial primary memory results when a corpus=all wiki supplement stalls", async () => {
456456
vi.useFakeTimers();
457457
try {
458458
let searchCalls = 0;
@@ -481,10 +481,28 @@ describe("memory tools", () => {
481481
});
482482
await vi.advanceTimersByTimeAsync(15_000);
483483
const stalledAllResult = await stalledAllResultPromise;
484-
expectUnavailableMemorySearchDetails(stalledAllResult.details, {
484+
const stalledDetails = stalledAllResult.details as {
485+
results: Array<{ corpus: string; path: string }>;
486+
partial?: boolean;
487+
error?: string;
488+
warning?: string;
489+
action?: string;
490+
debug?: { partial?: { failedPhase?: string; error?: string } };
491+
};
492+
expect(stalledDetails.results.map((entry) => [entry.corpus, entry.path])).toEqual([
493+
["memory", "MEMORY.md"],
494+
]);
495+
expect(stalledDetails.partial).toBe(true);
496+
expect(stalledDetails.error).toBe("memory_search timed out after 15s");
497+
expect(stalledDetails.warning).toBe(
498+
"Memory search returned partial results because a supplemental corpus did not finish.",
499+
);
500+
expect(stalledDetails.action).toBe(
501+
'Retry with corpus="memory" or corpus="wiki" for a narrower search, or increase plugins.entries.memory-core.config.tools.memorySearch.timeoutMs for slower hosts.',
502+
);
503+
expect(stalledDetails.debug?.partial).toEqual({
504+
failedPhase: "supplement",
485505
error: "memory_search timed out after 15s",
486-
warning: "Memory search is unavailable due to an embedding/provider error.",
487-
action: "Check embedding provider configuration and retry memory_search.",
488506
});
489507

490508
const memoryResult = await tool.execute("call_memory_after_stalled_wiki", {
@@ -531,8 +549,9 @@ describe("memory tools", () => {
531549
const stalledAllResult = await stalledAllResultPromise;
532550
expectUnavailableMemorySearchDetails(stalledAllResult.details, {
533551
error: "memory_search timed out after 15s",
534-
warning: "Memory search is unavailable due to an embedding/provider error.",
535-
action: "Check embedding provider configuration and retry memory_search.",
552+
warning: "Memory search timed out before the tool deadline.",
553+
action:
554+
"Retry with a narrower corpus/query or after Gateway pressure clears; if this recurs, increase plugins.entries.memory-core.config.tools.memorySearch.timeoutMs.",
536555
});
537556

538557
const wikiOnlyResult = await tool.execute("call_all_after_stalled_memory", {

‎extensions/memory-core/src/tools.shared.ts‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -127,16 +127,21 @@ export function buildMemorySearchUnavailableResult(
127127
) {
128128
const reason = (error ?? "memory search unavailable").trim() || "memory search unavailable";
129129
const isQuotaError = /insufficient_quota|quota|429/.test(normalizeLowercaseStringOrEmpty(reason));
130+
const isToolTimeout = /\bmemory_search timed out after \d+s\b/i.test(reason);
130131
const warning =
131132
overrides?.warning ??
132133
(isQuotaError
133134
? "Memory search is unavailable because the embedding provider quota is exhausted."
134-
: "Memory search is unavailable due to an embedding/provider error.");
135+
: isToolTimeout
136+
? "Memory search timed out before the tool deadline."
137+
: "Memory search is unavailable due to an embedding/provider error.");
135138
const action =
136139
overrides?.action ??
137140
(isQuotaError
138141
? "Top up or switch embedding provider, then retry memory_search."
139-
: "Check embedding provider configuration and retry memory_search.");
142+
: isToolTimeout
143+
? "Retry with a narrower corpus/query or after Gateway pressure clears; if this recurs, increase plugins.entries.memory-core.config.tools.memorySearch.timeoutMs."
144+
: "Check embedding provider configuration and retry memory_search.");
140145
return {
141146
results: [],
142147
disabled: true,

‎extensions/memory-core/src/tools.test.ts‎

Lines changed: 55 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -145,8 +145,9 @@ describe("memory_search unavailable payloads", () => {
145145
const result = await resultPromise;
146146
expectUnavailableMemorySearchDetails(result.details, {
147147
error: "memory_search timed out after 15s",
148-
warning: "Memory search is unavailable due to an embedding/provider error.",
149-
action: "Check embedding provider configuration and retry memory_search.",
148+
warning: "Memory search timed out before the tool deadline.",
149+
action:
150+
"Retry with a narrower corpus/query or after Gateway pressure clears; if this recurs, increase plugins.entries.memory-core.config.tools.memorySearch.timeoutMs.",
150151
});
151152
} finally {
152153
vi.useRealTimers();
@@ -171,23 +172,68 @@ describe("memory_search unavailable payloads", () => {
171172
const result = await resultPromise;
172173
expectUnavailableMemorySearchDetails(result.details, {
173174
error: "memory_search timed out after 15s",
174-
warning: "Memory search is unavailable due to an embedding/provider error.",
175-
action: "Check embedding provider configuration and retry memory_search.",
175+
warning: "Memory search timed out before the tool deadline.",
176+
action:
177+
"Retry with a narrower corpus/query or after Gateway pressure clears; if this recurs, increase plugins.entries.memory-core.config.tools.memorySearch.timeoutMs.",
176178
});
177179
// The deadline must abort the orphaned search, not just race past it.
178180
expect(searchSignal?.aborted).toBe(true);
179181
const cooldownResult = await tool.execute("search-cooldown", { query: "hello again" });
180182
expectUnavailableMemorySearchDetails(cooldownResult.details, {
181183
error: "memory_search timed out after 15s",
182-
warning: "Memory search is unavailable due to an embedding/provider error.",
183-
action: "Check embedding provider configuration and retry memory_search.",
184+
warning: "Memory search timed out before the tool deadline.",
185+
action:
186+
"Retry with a narrower corpus/query or after Gateway pressure clears; if this recurs, increase plugins.entries.memory-core.config.tools.memorySearch.timeoutMs.",
184187
});
185188
expect(searchCalls).toBe(1);
186189
} finally {
187190
vi.useRealTimers();
188191
}
189192
});
190193

194+
it("honors the configured memory_search tool deadline", async () => {
195+
vi.useFakeTimers();
196+
try {
197+
let searchSignal: AbortSignal | undefined;
198+
setMemorySearchImpl(async (opts) => {
199+
searchSignal = opts?.signal;
200+
return await new Promise(() => {});
201+
});
202+
const tool = createMemorySearchToolOrThrow({
203+
config: {
204+
agents: { list: [{ id: "main", default: true }] },
205+
plugins: {
206+
entries: {
207+
"memory-core": {
208+
config: {
209+
tools: {
210+
memorySearch: {
211+
timeoutMs: 5_000,
212+
},
213+
},
214+
},
215+
},
216+
},
217+
},
218+
},
219+
});
220+
221+
const resultPromise = tool.execute("configured-timeout", { query: "hello" });
222+
await vi.advanceTimersByTimeAsync(5_000);
223+
224+
const result = await resultPromise;
225+
expectUnavailableMemorySearchDetails(result.details, {
226+
error: "memory_search timed out after 5s",
227+
warning: "Memory search timed out before the tool deadline.",
228+
action:
229+
"Retry with a narrower corpus/query or after Gateway pressure clears; if this recurs, increase plugins.entries.memory-core.config.tools.memorySearch.timeoutMs.",
230+
});
231+
expect(searchSignal?.aborted).toBe(true);
232+
} finally {
233+
vi.useRealTimers();
234+
}
235+
});
236+
191237
it("keeps the timeout result when an abort-aware search rejects on abort", async () => {
192238
vi.useFakeTimers();
193239
try {
@@ -209,8 +255,9 @@ describe("memory_search unavailable payloads", () => {
209255
const result = await resultPromise;
210256
expectUnavailableMemorySearchDetails(result.details, {
211257
error: "memory_search timed out after 15s",
212-
warning: "Memory search is unavailable due to an embedding/provider error.",
213-
action: "Check embedding provider configuration and retry memory_search.",
258+
warning: "Memory search timed out before the tool deadline.",
259+
action:
260+
"Retry with a narrower corpus/query or after Gateway pressure clears; if this recurs, increase plugins.entries.memory-core.config.tools.memorySearch.timeoutMs.",
214261
});
215262
} finally {
216263
vi.useRealTimers();

‎extensions/memory-core/src/tools.ts‎

Lines changed: 118 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,9 @@ type MemorySearchToolResult =
4444
| (MemorySearchResult & { corpus: MemorySource })
4545
| MemoryCorpusSearchResult;
4646

47-
const MEMORY_SEARCH_TOOL_TIMEOUT_MS = 15_000;
47+
const DEFAULT_MEMORY_SEARCH_TOOL_TIMEOUT_MS = 15_000;
48+
const MIN_MEMORY_SEARCH_TOOL_TIMEOUT_MS = 1_000;
49+
const MAX_MEMORY_SEARCH_TOOL_TIMEOUT_MS = 120_000;
4850
const MEMORY_SEARCH_TOOL_COOLDOWN_MS = 60_000;
4951

5052
const memorySearchToolCooldowns = new Map<string, { until: number; error: string }>();
@@ -81,6 +83,23 @@ export const testing = {
8183
},
8284
} as const;
8385

86+
function clampMemorySearchToolTimeoutMs(value: number): number {
87+
return Math.min(
88+
MAX_MEMORY_SEARCH_TOOL_TIMEOUT_MS,
89+
Math.max(MIN_MEMORY_SEARCH_TOOL_TIMEOUT_MS, Math.round(value)),
90+
);
91+
}
92+
93+
function resolveMemorySearchToolTimeoutMs(cfg: OpenClawConfig): number {
94+
const pluginConfig = resolveMemoryCorePluginConfig(cfg);
95+
const toolsConfig = asRecord(pluginConfig?.tools);
96+
const memorySearchConfig = asRecord(toolsConfig?.memorySearch);
97+
const timeoutMs = memorySearchConfig?.timeoutMs;
98+
return typeof timeoutMs === "number" && Number.isFinite(timeoutMs) && timeoutMs > 0
99+
? clampMemorySearchToolTimeoutMs(timeoutMs)
100+
: DEFAULT_MEMORY_SEARCH_TOOL_TIMEOUT_MS;
101+
}
102+
84103
async function runMemorySearchToolWithDeadline<T>(params: {
85104
timeoutMs: number;
86105
run: (signal: AbortSignal) => Promise<T>;
@@ -185,6 +204,59 @@ function mergeMemorySearchCorpusResults(params: {
185204
return sortMemorySearchToolResults(selected).slice(0, params.maxResults);
186205
}
187206

207+
function buildPartialMemorySearchCorpusResult(params: {
208+
memoryResults: MemorySearchToolResult[];
209+
maxResults: number;
210+
provider?: string;
211+
model?: string;
212+
fallback: unknown;
213+
citations: unknown;
214+
mode?: string;
215+
debug?:
216+
| {
217+
backend: string;
218+
configuredMode?: string;
219+
effectiveMode?: string;
220+
fallback?: string;
221+
searchMs: number;
222+
hits: number;
223+
}
224+
| undefined;
225+
error: string;
226+
failedPhase: "memory" | "supplement";
227+
}) {
228+
const warning =
229+
params.failedPhase === "supplement"
230+
? "Memory search returned partial results because a supplemental corpus did not finish."
231+
: "Memory search returned partial results because the primary memory corpus did not finish.";
232+
const action =
233+
'Retry with corpus="memory" or corpus="wiki" for a narrower search, or increase plugins.entries.memory-core.config.tools.memorySearch.timeoutMs for slower hosts.';
234+
return jsonResult({
235+
results: mergeMemorySearchCorpusResults({
236+
memoryResults: params.memoryResults,
237+
supplementResults: [],
238+
maxResults: params.maxResults,
239+
balanceCorpora: false,
240+
}),
241+
provider: params.provider,
242+
model: params.model,
243+
fallback: params.fallback,
244+
citations: params.citations,
245+
mode: params.mode,
246+
partial: true,
247+
error: params.error,
248+
warning,
249+
action,
250+
debug: {
251+
...(params.debug ?? {}),
252+
partial: {
253+
failedPhase: params.failedPhase,
254+
error: params.error,
255+
},
256+
},
257+
});
258+
}
259+
188260
function isClosedMemoryStoreError(error: unknown): boolean {
189261
const message = formatErrorMessage(error).toLowerCase();
190262
return (
@@ -375,6 +447,27 @@ export function createMemorySearchTool(options: {
375447
requestedCorpus === "wiki" ? undefined : readMemorySearchToolCooldown(cooldownKey);
376448
let activeUnavailablePhase: "memory" | "supplement" | undefined;
377449
let failedUnavailablePhase: "memory" | "supplement" | undefined;
450+
let partialAllCorpusResult:
451+
| {
452+
memoryResults: MemorySearchToolResult[];
453+
maxResults: number;
454+
provider?: string;
455+
model?: string;
456+
fallback: unknown;
457+
citations: unknown;
458+
mode?: string;
459+
debug?:
460+
| {
461+
backend: string;
462+
configuredMode?: string;
463+
effectiveMode?: string;
464+
fallback?: string;
465+
searchMs: number;
466+
hits: number;
467+
}
468+
| undefined;
469+
}
470+
| undefined;
378471
const runUnavailablePhase = async <T>(
379472
phase: "memory" | "supplement",
380473
task: () => Promise<T>,
@@ -393,7 +486,7 @@ export function createMemorySearchTool(options: {
393486
};
394487

395488
const outcome = await runMemorySearchToolWithDeadline({
396-
timeoutMs: MEMORY_SEARCH_TOOL_TIMEOUT_MS,
489+
timeoutMs: resolveMemorySearchToolTimeoutMs(cfg),
397490
run: async (deadlineSignal) => {
398491
const { resolveMemoryBackendConfig } = await loadMemoryToolRuntime();
399492
const shouldQuerySupplements = requestedCorpus === "wiki" || requestedCorpus === "all";
@@ -555,6 +648,18 @@ export function createMemorySearchTool(options: {
555648
buildPausedMemoryIndexUnavailableResult(pausedIndexIdentityReason),
556649
);
557650
}
651+
if (requestedCorpus === "all" && surfacedMemoryResults.length > 0) {
652+
partialAllCorpusResult = {
653+
memoryResults: surfacedMemoryResults,
654+
maxResults: Math.max(1, maxResults ?? 10),
655+
provider,
656+
model,
657+
fallback,
658+
citations: citationsMode,
659+
mode: searchMode,
660+
debug: searchDebug,
661+
};
662+
}
558663
}
559664
const supplementResults = shouldQuerySupplements
560665
? await runUnavailablePhase(
@@ -590,6 +695,17 @@ export function createMemorySearchTool(options: {
590695
});
591696
if (outcome.status === "unavailable") {
592697
const unavailablePhase = failedUnavailablePhase ?? activeUnavailablePhase;
698+
if (
699+
requestedCorpus === "all" &&
700+
unavailablePhase === "supplement" &&
701+
partialAllCorpusResult
702+
) {
703+
return buildPartialMemorySearchCorpusResult({
704+
...partialAllCorpusResult,
705+
error: outcome.error,
706+
failedPhase: unavailablePhase,
707+
});
708+
}
593709
const shouldRecordCooldown =
594710
requestedCorpus !== "wiki" &&
595711
(requestedCorpus !== "all" || unavailablePhase === "memory");

0 commit comments

Comments
 (0)