fix(qa): keep searchable tool coverage report-only

vincentkoc · vincentkoc · commit 9f2c0a80b40f · 2026-05-21T23:55:35.000+08:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -16,6 +16,7 @@ Docs: https://docs.openclaw.ai
 ### Fixes
 
 - Providers/Ollama: treat Docker/OrbStack host aliases as local Ollama endpoints so `ollama-local` marker auth works when OpenClaw runs inside a VM/container and Ollama runs on the host. Fixes #84875.
+- QA-Lab: keep explicitly searchable/deferred OpenClaw dynamic tool rows report-only by default so tool-coverage gates do not treat mock discovery gaps as hard product failures. (#80319) Thanks @100yenadmin.
 - Agents: cap heartbeat model bleed context hints by the stored session window when runtime model metadata is unavailable, so overflow recovery advice does not suggest a larger window than the active session actually has.
 - Control UI/Web Push: use `https://openclaw.ai` as the generated default VAPID subject instead of the old localhost mailbox so iOS PWA push setup uses an Apple-acceptable subject when `OPENCLAW_VAPID_SUBJECT` is unset. Fixes #83134. (#83317) Thanks @IWhatsskill.
 - Agents/Pi: keep embedded session transcript writes from tripping false takeover detection after packaged npm onboarding agent turns.
diff --git a/extensions/qa-lab/src/runtime-tool-metadata.ts b/extensions/qa-lab/src/runtime-tool-metadata.ts
@@ -150,7 +150,10 @@ export function readRuntimeToolCoverageMetadata(params: {
   const capabilityLayer = capabilityLayerInput
     ? (capabilityLayerInput as QaRuntimeCapabilityLayer)
     : DEFAULT_CAPABILITY_LAYER_BY_BUCKET[bucket];
-  const required = readBoolean(toolCoverage?.required) ?? bucket !== "optional-profile-or-plugin";
+  const explicitSearchableDynamic = capabilityLayerInput === "openclaw-dynamic-searchable";
+  const required =
+    readBoolean(toolCoverage?.required) ??
+    (bucket !== "optional-profile-or-plugin" && !explicitSearchableDynamic);
   return {
     bucket,
     expectedLayer,
diff --git a/extensions/qa-lab/src/tool-coverage-report.test.ts b/extensions/qa-lab/src/tool-coverage-report.test.ts
@@ -223,6 +223,68 @@ describe("qa tool coverage report", () => {
     );
   });
 
+  it("keeps searchable OpenClaw dynamic tool rows report-only by default", () => {
+    const report = buildQaToolCoverageReport({
+      scenarios: [
+        makeScenario("tool-searchable-web-search", "web-search", {
+          toolName: "web_search",
+          toolCoverage: {
+            bucket: "openclaw-dynamic-integration",
+            expectedLayer: "openclaw-dynamic",
+            capabilityLayer: "openclaw-dynamic-searchable",
+          },
+        }),
+      ],
+      summary: {
+        scenarios: [
+          {
+            name: "tool web_search searchable",
+            status: "fail",
+            runtimeParity: {
+              scenarioId: "tool-searchable-web-search",
+              drift: "tool-call-shape",
+              driftDetails: "searchable discovery was report-only",
+              cells: {
+                pi: {
+                  runtime: "pi",
+                  transcriptBytes: "",
+                  toolCalls: [{ tool: "web_search", argsHash: "a", resultHash: "r" }],
+                  finalText: "",
+                  usage: { inputTokens: 0, outputTokens: 0, totalTokens: 0 },
+                  wallClockMs: 1,
+                  bootStateLines: [],
+                },
+                codex: {
+                  runtime: "codex",
+                  transcriptBytes: "",
+                  toolCalls: [],
+                  finalText: "",
+                  usage: { inputTokens: 0, outputTokens: 0, totalTokens: 0 },
+                  wallClockMs: 1,
+                  bootStateLines: [],
+                },
+              },
+            },
+          },
+        ],
+      },
+      generatedAt: "2026-05-10T00:00:00.000Z",
+    });
+
+    expect(report.pass).toBe(true);
+    expect(report.failures).toEqual([]);
+    expect(report.reportOnlyTools).toBe(1);
+    expect(report.passingTools).toBe(0);
+    expect(report.searchableDynamicTools).toBe(1);
+    expect(report.rows[0]).toEqual(
+      expect.objectContaining({
+        capabilityLayer: "openclaw-dynamic-searchable",
+        required: false,
+        drift: "tool-call-shape",
+      }),
+    );
+  });
+
   it("passes required OpenClaw dynamic tool coverage when both runtimes exercise the tool", () => {
     const report = buildQaToolCoverageReport({
       scenarios: [
diff --git a/extensions/qa-lab/src/tool-coverage-report.ts b/extensions/qa-lab/src/tool-coverage-report.ts
@@ -61,6 +61,7 @@ export type QaToolCoverageReport = {
   trackedTools: number;
   nativeWorkspaceTools: number;
   dynamicIntegrationTools: number;
+  searchableDynamicTools: number;
   optionalTools: number;
   passingTools: number;
   failingTools: number;
@@ -286,10 +287,14 @@ export function buildQaToolCoverageReport(params: {
     nativeWorkspaceTools: rows.filter((row) => row.bucket === "codex-native-workspace").length,
     dynamicIntegrationTools: rows.filter((row) => row.bucket === "openclaw-dynamic-integration")
       .length,
+    searchableDynamicTools: rows.filter(
+      (row) => row.capabilityLayer === "openclaw-dynamic-searchable",
+    ).length,
     optionalTools: rows.filter((row) => row.bucket === "optional-profile-or-plugin").length,
     passingTools: evaluated
       ? rows.filter(
           (row) =>
+            row.required &&
             !row.tracking &&
             row.pi === "pass" &&
             row.codex === "pass" &&
@@ -315,6 +320,7 @@ export function renderQaToolCoverageMarkdownReport(report: QaToolCoverageReport)
     `- Tracked issue rows: ${report.trackedTools}`,
     `- Codex-native workspace tools: ${report.nativeWorkspaceTools}`,
     `- OpenClaw dynamic integration tools: ${report.dynamicIntegrationTools}`,
+    `- Searchable/deferred dynamic tools: ${report.searchableDynamicTools}`,
     `- Optional/profile/plugin-dependent tools: ${report.optionalTools}`,
     `- Passing tools: ${report.passingTools}`,
     `- Failing tools: ${report.failingTools}`,
diff --git a/qa/scenarios/index.md b/qa/scenarios/index.md
@@ -31,7 +31,8 @@ Runtime parity tiers:
   default runtime-tool fixtures. OpenClaw dynamic integration tools in this
   tier are hard-gated by `openclaw qa coverage --tools --summary`; Codex-native
   workspace rows remain separately tracked until native/live behavior is the
-  asserted surface. Selected with
+  asserted surface. Rows that explicitly target searchable/deferred OpenClaw
+  dynamic loading stay report-only unless a fixture promotes them to required. Selected with
   `openclaw qa suite --runtime-pair pi,codex --runtime-parity-tier standard`
 - `optional`: profile-, plugin-, or external-service-dependent runtime-tool
   fixtures that stay out of the default release gate
