Skip to content

Commit a00b01f

Browse files
steipetesteipete
committed
fix: harden complex qa suite scenarios
1 parent b5d2bd6 commit a00b01f

18 files changed

Lines changed: 1658 additions & 194 deletions

‎extensions/memory-core/src/memory/index.test.ts‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import { mkdirSync, rmSync } from "node:fs";
22
import fs from "node:fs/promises";
33
import os from "node:os";
44
import path from "node:path";
5+
import { resolveSessionTranscriptsDirForAgent } from "openclaw/plugin-sdk/memory-core";
56
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
67
import {
78
clearMemoryEmbeddingProviders as clearRegistry,
@@ -377,4 +378,123 @@ describe("memory index", () => {
377378
const noResults = await manager.search("nonexistent_xyz_keyword");
378379
expect(noResults.length).toBe(0);
379380
});
381+
382+
it("prefers exact session transcript hits in FTS-only mode", async () => {
383+
forceNoProvider = true;
384+
const stateDir = path.join(workspaceDir, ".state-session-ranking");
385+
vi.stubEnv("OPENCLAW_STATE_DIR", stateDir);
386+
try {
387+
const cfg = createCfg({
388+
storePath: path.join(workspaceDir, "index-fts-session-ranking.sqlite"),
389+
sources: ["memory", "sessions"],
390+
sessionMemory: true,
391+
minScore: 0,
392+
hybrid: { enabled: true, vectorWeight: 0.7, textWeight: 0.3 },
393+
});
394+
const result = await getMemorySearchManager({ cfg, agentId: "main" });
395+
const manager = requireManager(result);
396+
managersForCleanup.add(manager);
397+
resetManagerForTest(manager);
398+
399+
const memoryPath = path.join(workspaceDir, "MEMORY.md");
400+
await fs.writeFile(memoryPath, "Project Nebula stale codename: ORBIT-9.\n", "utf8");
401+
const staleAt = new Date("2020-01-01T00:00:00.000Z");
402+
await fs.utimes(memoryPath, staleAt, staleAt);
403+
404+
const sessionsDir = resolveSessionTranscriptsDirForAgent("main");
405+
await fs.mkdir(sessionsDir, { recursive: true });
406+
const transcriptPath = path.join(sessionsDir, "session-ranking.jsonl");
407+
const now = Date.parse("2026-04-07T15:25:04.113Z");
408+
await fs.writeFile(
409+
transcriptPath,
410+
[
411+
JSON.stringify({
412+
type: "session",
413+
id: "session-ranking",
414+
timestamp: new Date(now - 60_000).toISOString(),
415+
}),
416+
JSON.stringify({
417+
type: "message",
418+
message: {
419+
role: "user",
420+
timestamp: new Date(now - 30_000).toISOString(),
421+
content: [{ type: "text", text: "What is the current Project Nebula codename?" }],
422+
},
423+
}),
424+
JSON.stringify({
425+
type: "message",
426+
message: {
427+
role: "assistant",
428+
timestamp: new Date(now).toISOString(),
429+
content: [{ type: "text", text: "The current Project Nebula codename is ORBIT-10." }],
430+
},
431+
}),
432+
].join("\n") + "\n",
433+
"utf8",
434+
);
435+
436+
await manager.sync({ reason: "test", force: true });
437+
const results = await manager.search("current Project Nebula codename ORBIT-10", {
438+
minScore: 0,
439+
maxResults: 3,
440+
});
441+
442+
expect(results[0]?.source).toBe("sessions");
443+
expect(results[0]?.snippet).toContain("ORBIT-10");
444+
} finally {
445+
vi.unstubAllEnvs();
446+
}
447+
});
448+
449+
it("bootstraps an empty index on first search so session transcript hits are available", async () => {
450+
forceNoProvider = true;
451+
const stateDir = path.join(workspaceDir, ".state-session-bootstrap");
452+
vi.stubEnv("OPENCLAW_STATE_DIR", stateDir);
453+
try {
454+
const cfg = createCfg({
455+
storePath: path.join(workspaceDir, "index-fts-session-bootstrap.sqlite"),
456+
sources: ["memory", "sessions"],
457+
sessionMemory: true,
458+
minScore: 0,
459+
hybrid: { enabled: true, vectorWeight: 0.7, textWeight: 0.3 },
460+
});
461+
const result = await getMemorySearchManager({ cfg, agentId: "main" });
462+
const manager = requireManager(result);
463+
managersForCleanup.add(manager);
464+
resetManagerForTest(manager);
465+
466+
const sessionsDir = resolveSessionTranscriptsDirForAgent("main");
467+
await fs.mkdir(sessionsDir, { recursive: true });
468+
const transcriptPath = path.join(sessionsDir, "session-bootstrap.jsonl");
469+
await fs.writeFile(
470+
transcriptPath,
471+
[
472+
JSON.stringify({
473+
type: "session",
474+
id: "session-bootstrap",
475+
timestamp: "2026-04-07T15:24:04.113Z",
476+
}),
477+
JSON.stringify({
478+
type: "message",
479+
message: {
480+
role: "assistant",
481+
timestamp: "2026-04-07T15:25:04.113Z",
482+
content: [{ type: "text", text: "The current Project Nebula codename is ORBIT-10." }],
483+
},
484+
}),
485+
].join("\n") + "\n",
486+
"utf8",
487+
);
488+
489+
const results = await manager.search("current Project Nebula codename ORBIT-10", {
490+
minScore: 0,
491+
maxResults: 3,
492+
});
493+
494+
expect(results[0]?.source).toBe("sessions");
495+
expect(results[0]?.snippet).toContain("ORBIT-10");
496+
} finally {
497+
vi.unstubAllEnvs();
498+
}
499+
});
380500
});

‎extensions/memory-core/src/memory/manager.ts‎

Lines changed: 39 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ import {
5454
runMemorySyncWithReadonlyRecovery,
5555
type MemoryReadonlyRecoveryState,
5656
} from "./manager-sync-control.js";
57+
import { applyTemporalDecayToHybridResults } from "./temporal-decay.js";
5758
const SNIPPET_MAX_CHARS = 700;
5859
const VECTOR_TABLE = "chunks_vec";
5960
const FTS_TABLE = "chunks_fts";
@@ -292,9 +293,21 @@ export class MemoryIndexManager extends MemoryManagerEmbeddingOps implements Mem
292293
sessionKey?: string;
293294
},
294295
): Promise<MemorySearchResult[]> {
296+
let hasIndexedContent = this.hasIndexedContent();
297+
if (!hasIndexedContent) {
298+
try {
299+
// A fresh process can receive its first search before background watch/session
300+
// syncs have built the index. Force one synchronous bootstrap so the first
301+
// lookup after restart does not fail closed with empty results.
302+
await this.sync({ reason: "search", force: true });
303+
} catch (err) {
304+
log.warn(`memory sync failed (search-bootstrap): ${String(err)}`);
305+
}
306+
hasIndexedContent = this.hasIndexedContent();
307+
}
295308
const preflight = resolveMemorySearchPreflight({
296309
query,
297-
hasIndexedContent: this.hasIndexedContent(),
310+
hasIndexedContent,
298311
});
299312
if (!preflight.shouldSearch) {
300313
return [];
@@ -328,17 +341,23 @@ export class MemoryIndexManager extends MemoryManagerEmbeddingOps implements Mem
328341
return [];
329342
}
330343

331-
// Extract keywords for better FTS matching on conversational queries
332-
// e.g., "that thing we discussed about the API" → ["discussed", "API"]
333-
const keywords = extractKeywords(cleaned, {
334-
ftsTokenizer: this.settings.store.fts.tokenizer,
335-
});
336-
const searchTerms = keywords.length > 0 ? keywords : [cleaned];
337-
338-
// Search with each keyword and merge results
339-
const resultSets = await Promise.all(
340-
searchTerms.map((term) => this.searchKeyword(term, candidates).catch(() => [])),
341-
);
344+
const fullQueryResults = await this.searchKeyword(cleaned, candidates).catch(() => []);
345+
const resultSets =
346+
fullQueryResults.length > 0
347+
? [fullQueryResults]
348+
: await Promise.all(
349+
// Fallback: broaden recall for conversational queries when the
350+
// exact AND query is too strict to return any results.
351+
(() => {
352+
const keywords = extractKeywords(cleaned, {
353+
ftsTokenizer: this.settings.store.fts.tokenizer,
354+
});
355+
const searchTerms = keywords.length > 0 ? keywords : [cleaned];
356+
return searchTerms.map((term) =>
357+
this.searchKeyword(term, candidates).catch(() => []),
358+
);
359+
})(),
360+
);
342361

343362
// Merge and deduplicate results, keeping highest score for each chunk
344363
const seenIds = new Map<string, (typeof resultSets)[0][0]>();
@@ -351,8 +370,14 @@ export class MemoryIndexManager extends MemoryManagerEmbeddingOps implements Mem
351370
}
352371
}
353372

354-
const merged = [...seenIds.values()].toSorted((a, b) => b.score - a.score);
355-
return this.selectScoredResults(merged, maxResults, minScore, 0);
373+
const merged = [...seenIds.values()];
374+
const decayed = await applyTemporalDecayToHybridResults({
375+
results: merged,
376+
temporalDecay: hybrid.temporalDecay,
377+
workspaceDir: this.workspaceDir,
378+
});
379+
const sorted = decayed.toSorted((a, b) => b.score - a.score);
380+
return this.selectScoredResults(sorted, maxResults, minScore, 0);
356381
}
357382

358383
// If FTS isn't available, hybrid mode cannot use keyword search; degrade to vector-only.

‎extensions/openai/image-generation-provider.test.ts‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ describe("openai image generation provider", () => {
3434
postJsonRequestMock.mockReset();
3535
assertOkOrThrowHttpErrorMock.mockClear();
3636
resolveProviderHttpRequestConfigMock.mockClear();
37+
vi.unstubAllEnvs();
3738
});
3839

3940
it("does not auto-allow local baseUrl overrides for image requests", async () => {
@@ -77,6 +78,88 @@ describe("openai image generation provider", () => {
7778
expect(result.images).toHaveLength(1);
7879
});
7980

81+
it("allows loopback image requests for the synthetic mock-openai provider", async () => {
82+
postJsonRequestMock.mockResolvedValue({
83+
response: {
84+
json: async () => ({
85+
data: [{ b64_json: Buffer.from("png-bytes").toString("base64") }],
86+
}),
87+
},
88+
release: vi.fn(async () => {}),
89+
});
90+
91+
const provider = buildOpenAIImageGenerationProvider();
92+
const result = await provider.generateImage({
93+
provider: "mock-openai",
94+
model: "gpt-image-1",
95+
prompt: "Draw a QA lighthouse",
96+
cfg: {
97+
models: {
98+
providers: {
99+
openai: {
100+
baseUrl: "http://127.0.0.1:44080/v1",
101+
models: [],
102+
},
103+
},
104+
},
105+
},
106+
});
107+
108+
expect(resolveProviderHttpRequestConfigMock).toHaveBeenCalledWith(
109+
expect.objectContaining({
110+
allowPrivateNetwork: true,
111+
}),
112+
);
113+
expect(postJsonRequestMock).toHaveBeenCalledWith(
114+
expect.objectContaining({
115+
url: "http://127.0.0.1:44080/v1/images/generations",
116+
allowPrivateNetwork: true,
117+
}),
118+
);
119+
expect(result.images).toHaveLength(1);
120+
});
121+
122+
it("allows loopback image requests for openai only inside the QA harness envelope", async () => {
123+
postJsonRequestMock.mockResolvedValue({
124+
response: {
125+
json: async () => ({
126+
data: [{ b64_json: Buffer.from("png-bytes").toString("base64") }],
127+
}),
128+
},
129+
release: vi.fn(async () => {}),
130+
});
131+
vi.stubEnv("OPENCLAW_QA_ALLOW_LOCAL_IMAGE_PROVIDER", "1");
132+
133+
const provider = buildOpenAIImageGenerationProvider();
134+
const result = await provider.generateImage({
135+
provider: "openai",
136+
model: "gpt-image-1",
137+
prompt: "Draw a QA lighthouse",
138+
cfg: {
139+
models: {
140+
providers: {
141+
openai: {
142+
baseUrl: "http://127.0.0.1:44080/v1",
143+
models: [],
144+
},
145+
},
146+
},
147+
},
148+
});
149+
150+
expect(resolveProviderHttpRequestConfigMock).toHaveBeenCalledWith(
151+
expect.objectContaining({
152+
allowPrivateNetwork: true,
153+
}),
154+
);
155+
expect(postJsonRequestMock).toHaveBeenCalledWith(
156+
expect.objectContaining({
157+
allowPrivateNetwork: true,
158+
}),
159+
);
160+
expect(result.images).toHaveLength(1);
161+
});
162+
80163
it("uses JSON image_url edits for input-image requests", async () => {
81164
postJsonRequestMock.mockResolvedValue({
82165
response: {

‎extensions/openai/image-generation-provider.ts‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,21 @@ const DEFAULT_OUTPUT_MIME = "image/png";
1414
const DEFAULT_SIZE = "1024x1024";
1515
const OPENAI_SUPPORTED_SIZES = ["1024x1024", "1024x1536", "1536x1024"] as const;
1616
const OPENAI_MAX_INPUT_IMAGES = 5;
17+
const MOCK_OPENAI_PROVIDER_ID = "mock-openai";
18+
19+
function shouldAllowPrivateImageEndpoint(req: {
20+
provider: string;
21+
cfg: { models?: { providers?: Record<string, { baseUrl?: string }> } };
22+
}) {
23+
if (req.provider === MOCK_OPENAI_PROVIDER_ID) {
24+
return true;
25+
}
26+
const baseUrl = resolveConfiguredOpenAIBaseUrl(req.cfg);
27+
if (!baseUrl.startsWith("http://127.0.0.1:") && !baseUrl.startsWith("http://localhost:")) {
28+
return false;
29+
}
30+
return process.env.OPENCLAW_QA_ALLOW_LOCAL_IMAGE_PROVIDER === "1";
31+
}
1732

1833
type OpenAIImageApiResponse = {
1934
data?: Array<{
@@ -68,6 +83,7 @@ export function buildOpenAIImageGenerationProvider(): ImageGenerationProvider {
6883
resolveProviderHttpRequestConfig({
6984
baseUrl: resolveConfiguredOpenAIBaseUrl(req.cfg),
7085
defaultBaseUrl: DEFAULT_OPENAI_IMAGE_BASE_URL,
86+
allowPrivateNetwork: shouldAllowPrivateImageEndpoint(req),
7187
defaultHeaders: {
7288
Authorization: `Bearer ${auth.apiKey}`,
7389
},

‎extensions/qa-lab/src/gateway-child.test.ts‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
22
import os from "node:os";
33
import path from "node:path";
44
import { afterEach, describe, expect, it } from "vitest";
5-
import { buildQaRuntimeEnv, resolveQaControlUiRoot } from "./gateway-child.js";
5+
import { __testing, buildQaRuntimeEnv, resolveQaControlUiRoot } from "./gateway-child.js";
66

77
const cleanups: Array<() => Promise<void>> = [];
88

@@ -33,6 +33,7 @@ describe("buildQaRuntimeEnv", () => {
3333
});
3434

3535
expect(env.OPENCLAW_TEST_FAST).toBe("1");
36+
expect(env.OPENCLAW_QA_ALLOW_LOCAL_IMAGE_PROVIDER).toBe("1");
3637
expect(env.OPENCLAW_ALLOW_SLOW_REPLY_TESTS).toBe("1");
3738
});
3839

@@ -93,6 +94,17 @@ describe("buildQaRuntimeEnv", () => {
9394
expect(env.OPENCLAW_LIVE_ANTHROPIC_KEYS).toBeUndefined();
9495
expect(env.OPENCLAW_LIVE_GEMINI_KEY).toBeUndefined();
9596
});
97+
98+
it("treats restart socket closures as retryable gateway call errors", () => {
99+
expect(__testing.isRetryableGatewayCallError("gateway closed (1006 abnormal closure)")).toBe(
100+
true,
101+
);
102+
expect(__testing.isRetryableGatewayCallError("gateway closed (1012 service restart)")).toBe(
103+
true,
104+
);
105+
expect(__testing.isRetryableGatewayCallError("service restart in progress")).toBe(true);
106+
expect(__testing.isRetryableGatewayCallError("permission denied")).toBe(false);
107+
});
96108
});
97109

98110
describe("resolveQaControlUiRoot", () => {

0 commit comments

Comments
 (0)