Skip to content

Commit cb42cc5

Browse files
steipetebdjben
steipete
and
bdjben
committed
fix(memory): scope qmd search deadline
Co-authored-by: Ben Badejo <[email protected]>
1 parent bab54ae commit cb42cc5

12 files changed

Lines changed: 1064 additions & 537 deletions

‎docs/concepts/memory-qmd.md‎

Lines changed: 3 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -264,20 +264,9 @@ does not run QMD vector status probes or embedding maintenance, and leaves
264264
semantic readiness checks to `vsearch` or `query` setups.
265265

266266
**Search times out?** Increase `memory.qmd.limits.timeoutMs` (default: 4000ms).
267-
Set to `120000` for slower hardware.
268-
269-
Manual smoke test after raising the timeout:
270-
271-
1. Set `memory.backend = "qmd"` and raise `memory.qmd.limits.timeoutMs` to a
272-
value above the old tool deadline, for example `45000`.
273-
2. Restart the gateway so the runtime picks up the config change.
274-
3. From a tool-enabled agent session, run `memory_search` with a query that is
275-
known to take longer than 15 seconds on that machine.
276-
4. Inspect the gateway/backend logs for the search window. The old failure
277-
string, `memory_search timed out after 15s`, should not appear. A successful
278-
search may return results or an empty result set. If the search still exceeds
279-
the configured budget, the unavailable payload should name the configured
280-
timeout, for example `memory_search timed out after 45s`.
267+
Set to `120000` for slower hardware. This limit also applies to QMD's own search
268+
commands during agent `memory_search` calls; setup, sync, builtin fallback, and
269+
supplemental corpus work keep their own shorter deadlines.
281270

282271
**Empty results in group chats?** Check `memory.qmd.scope` -- the default only
283272
allows direct and channel sessions.

‎docs/reference/memory-config.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -575,7 +575,7 @@ QMD model overrides stay on the QMD side, not OpenClaw config. If you need to ov
575575
| `limits.maxResults` | `number` | `6` | Max search results |
576576
| `limits.maxSnippetChars` | `number` | -- | Clamp snippet length |
577577
| `limits.maxInjectedChars` | `number` | -- | Clamp total injected chars |
578-
| `limits.timeoutMs` | `number` | `4000` | Search timeout, also used as the `memory_search` tool deadline for QMD-backed searches |
578+
| `limits.timeoutMs` | `number` | `4000` | QMD command timeout during QMD-backed search, including `memory_search`; setup, sync, builtin fallback, and supplemental work keep the default tool deadline |
579579
</Accordion>
580580
<Accordion title="Scope">
581581
Controls which sessions can receive QMD search results. Same schema as [`session.sendPolicy`](/gateway/config-agents#session):

‎extensions/memory-core/src/memory-tool-manager.test-mocks.ts‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
// Memory Core plugin module implements memory tool manager mock behavior.
22
import type { MemorySearchRuntimeDebug } from "openclaw/plugin-sdk/memory-core-host-runtime-files";
33
import { vi } from "vitest";
4+
import {
5+
MEMORY_SEARCH_DEADLINE_CONTROL,
6+
type MemorySearchDeadlineAction,
7+
} from "./memory/search-deadline.js";
48

59
type SearchImpl = (opts?: {
610
maxResults?: number;
@@ -9,6 +13,7 @@ type SearchImpl = (opts?: {
913
qmdSearchModeOverride?: "query" | "search" | "vsearch";
1014
onDebug?: (debug: MemorySearchRuntimeDebug) => void;
1115
signal?: AbortSignal;
16+
[MEMORY_SEARCH_DEADLINE_CONTROL]?: (action: MemorySearchDeadlineAction) => void;
1217
}) => Promise<unknown[]>;
1318
export type MemoryReadParams = { relPath: string; from?: number; lines?: number };
1419
type MemoryReadResult = {
@@ -26,6 +31,7 @@ let resolvedBackend: MemoryBackend | undefined;
2631
let workspaceDir = "/workspace";
2732
let customStatus: Record<string, unknown> | undefined;
2833
let searchImpl: SearchImpl = async () => [];
34+
let closeImpl: () => Promise<void> = async () => {};
2935
let getManagerImpl:
3036
| ((params: { cfg?: unknown; agentId?: string; purpose?: string }) => Promise<{
3137
manager?: unknown;
@@ -58,7 +64,7 @@ const stubManager = {
5864
}),
5965
sync: vi.fn(),
6066
probeVectorAvailability: vi.fn(async () => true),
61-
close: vi.fn(async () => {}),
67+
close: vi.fn(async () => await closeImpl()),
6268
};
6369

6470
const getMemorySearchManagerMock = vi.fn(
@@ -102,6 +108,10 @@ export function setMemorySearchImpl(next: SearchImpl): void {
102108
searchImpl = next;
103109
}
104110

111+
export function setMemoryCloseImpl(next: () => Promise<void>): void {
112+
closeImpl = next;
113+
}
114+
105115
export function setMemorySearchManagerImpl(
106116
next: (params: { cfg?: unknown; agentId?: string; purpose?: string }) => Promise<{
107117
manager?: unknown;
@@ -128,6 +138,7 @@ export function resetMemoryToolMockState(overrides?: {
128138
customStatus = undefined;
129139
getManagerImpl = undefined;
130140
searchImpl = overrides?.searchImpl ?? (async () => []);
141+
closeImpl = async () => {};
131142
readFileImpl =
132143
overrides?.readFileImpl ??
133144
(async (params: MemoryReadParams) => ({

‎extensions/memory-core/src/memory/qmd-manager.test.ts‎

Lines changed: 39 additions & 112 deletions
Original file line numberDiff line numberDiff line change
@@ -212,6 +212,7 @@ import {
212212
} from "../dreaming-state.js";
213213
import { resolveQmdSessionArtifactIdentity } from "../qmd-session-artifacts.js";
214214
import { QmdMemoryManager, resolveQmdMcporterSearchProcessTimeoutMs } from "./qmd-manager.js";
215+
import { MEMORY_SEARCH_DEADLINE_CONTROL } from "./search-deadline.js";
215216

216217
const spawnMock = mockedSpawn as unknown as Mock;
217218
const originalPath = process.env.PATH;
@@ -292,117 +293,6 @@ describe("QmdMemoryManager", () => {
292293
expect(mockMessages(mock).join("\n")).not.toContain(text);
293294
}
294295

295-
function createClosableQmdManagerForTest(params?: {
296-
mode?: "full" | "status" | "cli";
297-
pendingUpdate?: Promise<void> | null;
298-
queuedForcedUpdate?: Promise<void> | null;
299-
}): QmdMemoryManager {
300-
const manager = Object.create(QmdMemoryManager.prototype) as QmdMemoryManager;
301-
const mutable = manager as unknown as {
302-
closed: boolean;
303-
resolveCloseSignal: () => void;
304-
updateTimer: NodeJS.Timeout | null;
305-
embedTimer: NodeJS.Timeout | null;
306-
watchTimer: NodeJS.Timeout | null;
307-
watcher: { close: () => Promise<void> } | null;
308-
queuedForcedRuns: number;
309-
pendingUpdate: Promise<void> | null;
310-
queuedForcedUpdate: Promise<void> | null;
311-
mode: "full" | "status" | "cli";
312-
db: { close: () => void } | null;
313-
};
314-
mutable.closed = false;
315-
mutable.resolveCloseSignal = vi.fn();
316-
mutable.updateTimer = null;
317-
mutable.embedTimer = null;
318-
mutable.watchTimer = null;
319-
mutable.watcher = null;
320-
mutable.queuedForcedRuns = 0;
321-
mutable.pendingUpdate = params?.pendingUpdate ?? null;
322-
mutable.queuedForcedUpdate = params?.queuedForcedUpdate ?? null;
323-
mutable.mode = params?.mode ?? "cli";
324-
mutable.db = null;
325-
return manager;
326-
}
327-
328-
it("clears qmd CLI close timeout timers when update waits settle first", async () => {
329-
vi.useFakeTimers();
330-
try {
331-
const manager = createClosableQmdManagerForTest({
332-
pendingUpdate: Promise.resolve(),
333-
queuedForcedUpdate: Promise.resolve(),
334-
});
335-
336-
await manager.close();
337-
await vi.advanceTimersByTimeAsync(5_000);
338-
339-
expectMockMessageNotContains(logWarnMock, "qmd close timed out");
340-
expect(vi.getTimerCount()).toBe(0);
341-
} finally {
342-
vi.useRealTimers();
343-
}
344-
});
345-
346-
it("uses one qmd CLI close timeout across pending and queued update waits", async () => {
347-
vi.useFakeTimers();
348-
try {
349-
const manager = createClosableQmdManagerForTest({
350-
pendingUpdate: new Promise(() => {}),
351-
queuedForcedUpdate: new Promise(() => {}),
352-
});
353-
let closed = false;
354-
355-
const closePromise = manager.close().then(() => {
356-
closed = true;
357-
});
358-
359-
await vi.advanceTimersByTimeAsync(4_999);
360-
await Promise.resolve();
361-
expect(closed).toBe(false);
362-
363-
await vi.advanceTimersByTimeAsync(1);
364-
await closePromise;
365-
366-
expect(closed).toBe(true);
367-
expectMockMessageContains(
368-
logWarnMock,
369-
"qmd close timed out waiting for pending update and queued forced update after 5000ms",
370-
);
371-
} finally {
372-
vi.useRealTimers();
373-
}
374-
});
375-
376-
it("waits for pending update cleanup without timeout for full qmd managers", async () => {
377-
vi.useFakeTimers();
378-
try {
379-
let resolvePending!: () => void;
380-
const manager = createClosableQmdManagerForTest({
381-
mode: "full",
382-
pendingUpdate: new Promise<void>((resolve) => {
383-
resolvePending = resolve;
384-
}),
385-
});
386-
let closed = false;
387-
388-
const closePromise = manager.close().then(() => {
389-
closed = true;
390-
});
391-
392-
await vi.advanceTimersByTimeAsync(5_000);
393-
await Promise.resolve();
394-
expect(closed).toBe(false);
395-
396-
resolvePending();
397-
await closePromise;
398-
399-
expect(closed).toBe(true);
400-
expectMockMessageNotContains(logWarnMock, "qmd close timed out");
401-
} finally {
402-
vi.useRealTimers();
403-
}
404-
});
405-
406296
it("caps mcporter search process timeout grace", () => {
407297
expect(resolveQmdMcporterSearchProcessTimeoutMs(1_000)).toBe(5_000);
408298
expect(resolveQmdMcporterSearchProcessTimeoutMs(10_000)).toBe(12_000);
@@ -529,7 +419,9 @@ describe("QmdMemoryManager", () => {
529419
const { manager } = await createManager();
530420
spawnMock.mockClear();
531421
let searchAttempts = 0;
422+
const events: string[] = [];
532423
spawnMock.mockImplementation((_cmd: string, args: string[]) => {
424+
events.push(`command:${args[0]}${args[1] ? `:${args[1]}` : ""}`);
533425
if (args[0] === "query" || args[0] === "search" || args[0] === "vsearch") {
534426
const child = createMockChild({ autoClose: false });
535427
searchAttempts += 1;
@@ -549,11 +441,38 @@ describe("QmdMemoryManager", () => {
549441
onDebug: (entry) => {
550442
debug.push(entry);
551443
},
444+
[MEMORY_SEARCH_DEADLINE_CONTROL]: (action) => {
445+
events.push(`phase:${action}`);
446+
},
552447
});
553448

554449
expect(searchAttempts).toBe(2);
555450
expect(countQmdCommand((args) => args[0] === "collection" && args[1] === "list")).toBe(1);
556451
expect(debug.at(-1)?.qmd?.collectionValidation?.cacheState).toBe("bypass-force");
452+
expect(events.filter((event) => event.startsWith("phase:"))).toEqual([
453+
"phase:pause",
454+
"phase:resume",
455+
"phase:pause",
456+
"phase:resume",
457+
]);
458+
const isSearchCommand = (event: string) =>
459+
["command:query:", "command:search:", "command:vsearch:"].some((prefix) =>
460+
event.startsWith(prefix),
461+
);
462+
const firstSearch = events.findIndex(isSearchCommand);
463+
const firstSearchEnd = events.indexOf("phase:resume");
464+
const collectionRepair = events.findIndex(
465+
(event, index) => index > firstSearchEnd && event.startsWith("command:collection:"),
466+
);
467+
const retryStart = events.indexOf("phase:pause", firstSearchEnd + 1);
468+
const retrySearch = events.findIndex(
469+
(event, index) => index > firstSearch && isSearchCommand(event),
470+
);
471+
expect(events.indexOf("phase:pause")).toBeLessThan(firstSearch);
472+
expect(firstSearch).toBeLessThan(firstSearchEnd);
473+
expect(firstSearchEnd).toBeLessThan(collectionRepair);
474+
expect(collectionRepair).toBeLessThan(retryStart);
475+
expect(retryStart).toBeLessThan(retrySearch);
557476
});
558477

559478
it("reuses persisted qmd multi-collection support probe across managers", async () => {
@@ -4107,9 +4026,11 @@ describe("QmdMemoryManager", () => {
41074026
},
41084027
} as OpenClawConfig;
41094028

4029+
const commandPhases: string[] = [];
41104030
spawnMock.mockImplementation((cmd: string, args: string[]) => {
41114031
const child = createMockChild({ autoClose: false });
41124032
if (isMcporterCommand(cmd) && args[0] === "call") {
4033+
expect(commandPhases).toEqual(["pause"]);
41134034
// Verify it calls qmd.query (v2) not qmd.deep_search (v1)
41144035
expect(args[1]).toBe("qmd.query");
41154036
const callArgs = JSON.parse(args[args.indexOf("--args") + 1]);
@@ -4134,7 +4055,13 @@ describe("QmdMemoryManager", () => {
41344055
});
41354056

41364057
const { manager } = await createManager();
4137-
await manager.search("hello", { sessionKey: "agent:main:slack:dm:u123" });
4058+
await manager.search("hello", {
4059+
sessionKey: "agent:main:slack:dm:u123",
4060+
[MEMORY_SEARCH_DEADLINE_CONTROL]: (action) => {
4061+
commandPhases.push(action);
4062+
},
4063+
});
4064+
expect(commandPhases).toEqual(["pause", "resume"]);
41384065
await manager.close();
41394066
});
41404067

0 commit comments

Comments
 (0)