Skip to content

Commit 9e2bd8b

Browse files
steipeteadone0
steipete
and
adone0
committed
fix(memory): fail open when embedding recall stalls
Preserve custom OpenAI-compatible memory embedding provider ids from #81170. Fixes #47884. Fixes #49524. Refs #56532. Co-authored-by: adone0 <[email protected]>
1 parent 2d3fa48 commit 9e2bd8b

11 files changed

Lines changed: 711 additions & 164 deletions

‎docs/reference/memory-config.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,11 @@ explicitly to use Gemini, Voyage, Mistral, DeepInfra, Bedrock, GitHub Copilot,
5858
Ollama, a local GGUF model, or an OpenAI-compatible `/v1/embeddings` endpoint.
5959
Legacy configs that still say `provider: "auto"` resolve to `openai`.
6060

61+
If OpenAI embeddings are unreachable from your network, memory recall fails open
62+
instead of blocking the turn. Set the existing `memorySearch.provider` field to a
63+
reachable local, Ollama, regional, or OpenAI-compatible provider to restore
64+
semantic ranking.
65+
6166
### Custom provider ids
6267

6368
`memorySearch.provider` can point at a custom `models.providers.<id>` entry for memory-specific provider adapters such as `ollama`, or for OpenAI-compatible model APIs such as `openai-responses` / `openai-completions`. OpenClaw resolves that provider's `api` owner for the embedding adapter while preserving the custom provider id for endpoint, auth, and model-prefix handling. This lets multi-GPU or multi-host setups dedicate memory embeddings to a specific local endpoint:

‎extensions/memory-core/src/memory-tool-manager-mock.ts‎

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,12 @@ let backend: MemoryBackend = "builtin";
2323
let workspaceDir = "/workspace";
2424
let customStatus: Record<string, unknown> | undefined;
2525
let searchImpl: SearchImpl = async () => [];
26+
let getManagerImpl:
27+
| ((params: { cfg?: unknown; agentId?: string }) => Promise<{
28+
manager?: unknown;
29+
error?: string;
30+
}>)
31+
| undefined;
2632
let readFileImpl: (params: MemoryReadParams) => Promise<MemoryReadResult> = async (params) => ({
2733
text: "",
2834
path: params.relPath,
@@ -52,9 +58,9 @@ const stubManager = {
5258
close: vi.fn(),
5359
};
5460

55-
const getMemorySearchManagerMock = vi.fn(async (_params: { cfg?: unknown; agentId?: string }) => ({
56-
manager: stubManager,
57-
}));
61+
const getMemorySearchManagerMock = vi.fn(async (params: { cfg?: unknown; agentId?: string }) =>
62+
getManagerImpl ? await getManagerImpl(params) : { manager: stubManager },
63+
);
5864
const readAgentMemoryFileMock = vi.fn(
5965
async (params: MemoryReadParams) => await readFileImpl(params),
6066
);
@@ -84,6 +90,15 @@ export function setMemorySearchImpl(next: SearchImpl): void {
8490
searchImpl = next;
8591
}
8692

93+
export function setMemorySearchManagerImpl(
94+
next: (params: { cfg?: unknown; agentId?: string }) => Promise<{
95+
manager?: unknown;
96+
error?: string;
97+
}>,
98+
): void {
99+
getManagerImpl = next;
100+
}
101+
87102
export function setMemoryReadFileImpl(
88103
next: (params: MemoryReadParams) => Promise<MemoryReadResult>,
89104
): void {
@@ -98,6 +113,7 @@ export function resetMemoryToolMockState(overrides?: {
98113
backend = overrides?.backend ?? "builtin";
99114
workspaceDir = "/workspace";
100115
customStatus = undefined;
116+
getManagerImpl = undefined;
101117
searchImpl = overrides?.searchImpl ?? (async () => []);
102118
readFileImpl =
103119
overrides?.readFileImpl ??

‎extensions/memory-core/src/tools.citations.test.ts‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import {
1717
type MemoryReadParams,
1818
} from "./memory-tool-manager-mock.js";
1919
import { createMemoryCoreTestHarness } from "./test-helpers.js";
20+
import { testing as memoryToolsTesting } from "./tools.js";
2021
import {
2122
asOpenClawConfig,
2223
createAutoCitationsMemorySearchTool,
@@ -51,6 +52,7 @@ async function waitFor<T>(task: () => Promise<T>, timeoutMs = 1500): Promise<T>
5152

5253
beforeEach(() => {
5354
clearMemoryPluginState();
55+
memoryToolsTesting.resetMemorySearchToolCooldowns();
5456
resetMemoryToolMockState({
5557
backend: "builtin",
5658
searchImpl: async () => [
@@ -447,6 +449,105 @@ describe("memory tools", () => {
447449
expect(getMemorySearchManagerMockCalls()).toBe(1);
448450
});
449451

452+
it("does not cooldown primary memory when a corpus=all wiki supplement stalls", async () => {
453+
vi.useFakeTimers();
454+
try {
455+
let searchCalls = 0;
456+
setMemorySearchImpl(async () => {
457+
searchCalls += 1;
458+
return [
459+
{
460+
path: "MEMORY.md",
461+
startLine: 5,
462+
endLine: 7,
463+
score: 0.9,
464+
snippet: "@@ -5,3 @@\nAssistant: noted",
465+
source: "memory" as const,
466+
},
467+
];
468+
});
469+
registerMemoryCorpusSupplement("memory-wiki", {
470+
search: async () => await new Promise(() => undefined),
471+
get: async () => null,
472+
});
473+
474+
const tool = createMemorySearchToolOrThrow();
475+
const stalledAllResultPromise = tool.execute("call_all_stalled_wiki", {
476+
query: "alpha",
477+
corpus: "all",
478+
});
479+
await vi.advanceTimersByTimeAsync(15_000);
480+
const stalledAllResult = await stalledAllResultPromise;
481+
expectUnavailableMemorySearchDetails(stalledAllResult.details, {
482+
error: "memory_search timed out after 15s",
483+
warning: "Memory search is unavailable due to an embedding/provider error.",
484+
action: "Check embedding provider configuration and retry memory_search.",
485+
});
486+
487+
const memoryResult = await tool.execute("call_memory_after_stalled_wiki", {
488+
query: "alpha",
489+
});
490+
const details = memoryResult.details as { results: Array<{ corpus: string; path: string }> };
491+
expect(details.results.map((entry) => [entry.corpus, entry.path])).toEqual([
492+
["memory", "MEMORY.md"],
493+
]);
494+
expect(searchCalls).toBe(2);
495+
} finally {
496+
vi.useRealTimers();
497+
}
498+
});
499+
500+
it("cooldowns primary memory when corpus=all memory search stalls", async () => {
501+
vi.useFakeTimers();
502+
try {
503+
let searchCalls = 0;
504+
setMemorySearchImpl(async () => {
505+
searchCalls += 1;
506+
return await new Promise(() => undefined);
507+
});
508+
registerMemoryCorpusSupplement("memory-wiki", {
509+
search: async () => [
510+
{
511+
corpus: "wiki",
512+
path: "entities/alpha.md",
513+
title: "Alpha",
514+
kind: "entity",
515+
score: 4,
516+
snippet: "Alpha wiki entry",
517+
},
518+
],
519+
get: async () => null,
520+
});
521+
522+
const tool = createMemorySearchToolOrThrow();
523+
const stalledAllResultPromise = tool.execute("call_all_stalled_memory", {
524+
query: "alpha",
525+
corpus: "all",
526+
});
527+
await vi.advanceTimersByTimeAsync(15_000);
528+
const stalledAllResult = await stalledAllResultPromise;
529+
expectUnavailableMemorySearchDetails(stalledAllResult.details, {
530+
error: "memory_search timed out after 15s",
531+
warning: "Memory search is unavailable due to an embedding/provider error.",
532+
action: "Check embedding provider configuration and retry memory_search.",
533+
});
534+
535+
const wikiOnlyResult = await tool.execute("call_all_after_stalled_memory", {
536+
query: "alpha",
537+
corpus: "all",
538+
});
539+
const details = wikiOnlyResult.details as {
540+
results: Array<{ corpus: string; path: string }>;
541+
};
542+
expect(details.results.map((entry) => [entry.corpus, entry.path])).toEqual([
543+
["wiki", "entities/alpha.md"],
544+
]);
545+
expect(searchCalls).toBe(1);
546+
} finally {
547+
vi.useRealTimers();
548+
}
549+
});
550+
450551
it("falls back to a wiki corpus supplement for memory_get corpus=all", async () => {
451552
setMemoryReadFileImpl(async () => {
452553
throw new Error("path required");

‎extensions/memory-core/src/tools.test.ts‎

Lines changed: 54 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,9 @@ import {
66
resetMemoryToolMockState,
77
setMemoryBackend,
88
setMemorySearchImpl,
9+
setMemorySearchManagerImpl,
910
} from "./memory-tool-manager-mock.js";
10-
import { createMemorySearchTool } from "./tools.js";
11+
import { createMemorySearchTool, testing as memoryToolsTesting } from "./tools.js";
1112
import { MemoryGetSchema, MemorySearchSchema } from "./tools.shared.js";
1213
import {
1314
asOpenClawConfig,
@@ -56,6 +57,7 @@ describe("memory tool schemas", () => {
5657
describe("memory_search unavailable payloads", () => {
5758
beforeEach(() => {
5859
resetMemoryToolMockState({ searchImpl: async () => [] });
60+
memoryToolsTesting.resetMemorySearchToolCooldowns();
5961
});
6062

6163
it("rejects fractional maxResults before searching", async () => {
@@ -128,6 +130,57 @@ describe("memory_search unavailable payloads", () => {
128130
});
129131
});
130132

133+
it("returns unavailable metadata when manager setup does not settle", async () => {
134+
vi.useFakeTimers();
135+
try {
136+
setMemorySearchManagerImpl(async () => await new Promise(() => undefined));
137+
const tool = createMemorySearchToolOrThrow();
138+
139+
const resultPromise = tool.execute("manager-timeout", { query: "hello" });
140+
await vi.advanceTimersByTimeAsync(15_000);
141+
142+
const result = await resultPromise;
143+
expectUnavailableMemorySearchDetails(result.details, {
144+
error: "memory_search timed out after 15s",
145+
warning: "Memory search is unavailable due to an embedding/provider error.",
146+
action: "Check embedding provider configuration and retry memory_search.",
147+
});
148+
} finally {
149+
vi.useRealTimers();
150+
}
151+
});
152+
153+
it("returns unavailable metadata when memory search does not settle", async () => {
154+
vi.useFakeTimers();
155+
try {
156+
let searchCalls = 0;
157+
setMemorySearchImpl(async () => {
158+
searchCalls += 1;
159+
return await new Promise(() => undefined);
160+
});
161+
const tool = createMemorySearchToolOrThrow();
162+
163+
const resultPromise = tool.execute("search-timeout", { query: "hello" });
164+
await vi.advanceTimersByTimeAsync(15_000);
165+
166+
const result = await resultPromise;
167+
expectUnavailableMemorySearchDetails(result.details, {
168+
error: "memory_search timed out after 15s",
169+
warning: "Memory search is unavailable due to an embedding/provider error.",
170+
action: "Check embedding provider configuration and retry memory_search.",
171+
});
172+
const cooldownResult = await tool.execute("search-cooldown", { query: "hello again" });
173+
expectUnavailableMemorySearchDetails(cooldownResult.details, {
174+
error: "memory_search timed out after 15s",
175+
warning: "Memory search is unavailable due to an embedding/provider error.",
176+
action: "Check embedding provider configuration and retry memory_search.",
177+
});
178+
expect(searchCalls).toBe(1);
179+
} finally {
180+
vi.useRealTimers();
181+
}
182+
});
183+
131184
it("re-resolves the manager once when a cached sqlite handle was closed", async () => {
132185
let searchCalls = 0;
133186
setMemorySearchImpl(async () => {

0 commit comments

Comments
 (0)