fix(qa): sanitize ux evidence artifact paths

vincentkoc · vincentkoc · commit f04c3d6575e1 · 2026-06-20T03:46:06.000+02:00
diff --git a/scripts/qa/ux-matrix-evidence-producer.ts b/scripts/qa/ux-matrix-evidence-producer.ts
@@ -112,6 +112,33 @@ function relativeToArtifactBase(artifactBase: string, filePath: string) {
   return path.relative(artifactBase, filePath).split(path.sep).join("/");
 }
 
+function sanitizeArtifactText(
+  value: string,
+  params: {
+    artifactBase?: string;
+    repoRoot: string;
+  },
+) {
+  const roots = [
+    { from: path.resolve(params.repoRoot), to: "<repo-root>" },
+    { from: pathToFileURL(path.resolve(params.repoRoot)).href, to: "file://<repo-root>" },
+    ...(params.artifactBase
+      ? [
+          { from: path.resolve(params.artifactBase), to: "<artifact-base>" },
+          {
+            from: pathToFileURL(path.resolve(params.artifactBase)).href,
+            to: "file://<artifact-base>",
+          },
+        ]
+      : []),
+    { from: os.homedir(), to: "<home>" },
+    { from: pathToFileURL(os.homedir()).href, to: "file://<home>" },
+  ].filter((entry) => entry.from && entry.from !== path.parse(entry.from).root);
+  return roots
+    .toSorted((a, b) => b.from.length - a.from.length)
+    .reduce((text, entry) => text.replaceAll(entry.from, entry.to), value);
+}
+
 function buildExecution(params: {
   artifacts: MatrixCell["artifacts"];
   source: string;
@@ -195,6 +222,7 @@ function buildEvidenceSummary(params: {
 
 async function runCommandForCell(params: {
   args: string[];
+  artifactBase: string;
   command: string;
   cwd: string;
   logPath: string;
@@ -209,13 +237,22 @@ async function runCommandForCell(params: {
       env: process.env,
       maxBuffer: 1024 * 1024,
     });
-    await writeText(params.logPath, `$ ${commandLine}\n${stdout}${stderr}`);
+    await writeText(
+      params.logPath,
+      sanitizeArtifactText(`$ ${commandLine}\n${stdout}${stderr}`, {
+        artifactBase: params.artifactBase,
+        repoRoot: params.cwd,
+      }),
+    );
     return {
       status: "pass" as const,
       wallMs: Date.now() - startedAt,
     };
   } catch (error) {
-    const details = error instanceof Error ? error.message : String(error);
+    const details = sanitizeArtifactText(error instanceof Error ? error.message : String(error), {
+      artifactBase: params.artifactBase,
+      repoRoot: params.cwd,
+    });
     await writeText(params.logPath, `$ ${commandLine}\nblocked: ${details}\n`);
     return {
       failureReason: details,
@@ -242,6 +279,7 @@ async function captureControlUiScreenshot(params: {
   artifactBase: string;
   htmlPath: string;
   logPath: string;
+  repoRoot: string;
   screenshotPath: string;
 }) {
   const startedAt = Date.now();
@@ -264,7 +302,10 @@ async function captureControlUiScreenshot(params: {
       wallMs: Date.now() - startedAt,
     };
   } catch (error) {
-    const details = error instanceof Error ? error.message : String(error);
+    const details = sanitizeArtifactText(error instanceof Error ? error.message : String(error), {
+      artifactBase: params.artifactBase,
+      repoRoot: params.repoRoot,
+    });
     await writeText(params.logPath, `blocked: ${details}\n`);
     return {
       failureReason: details,
@@ -363,8 +404,10 @@ async function writeProducerArtifactFixtureHtml(params: {
 }
 
 async function captureProducerArtifactFixtureProof(params: {
+  artifactBase: string;
   htmlPath: string;
   logPath: string;
+  repoRoot: string;
   screenshotPath: string;
   skipVisualProof: boolean;
   videoPath: string;
@@ -423,7 +466,10 @@ async function captureProducerArtifactFixtureProof(params: {
       wallMs: Date.now() - startedAt,
     };
   } catch (error) {
-    const details = error instanceof Error ? error.message : String(error);
+    const details = sanitizeArtifactText(error instanceof Error ? error.message : String(error), {
+      artifactBase: params.artifactBase,
+      repoRoot: params.repoRoot,
+    });
     await writeText(params.logPath, `blocked: ${details}\n`);
     return {
       failureReason: details,
@@ -494,8 +540,9 @@ export async function runUxMatrixEvidenceProducer(options: ProducerOptions) {
     "logs.txt",
   );
   const cliResult = await runCommandForCell({
-    command: process.execPath,
     args: ["openclaw.mjs", "--help"],
+    artifactBase: options.artifactBase,
+    command: process.execPath,
     cwd: options.repoRoot,
     logPath: cliLogPath,
     timeoutMs: 30_000,
@@ -515,6 +562,7 @@ export async function runUxMatrixEvidenceProducer(options: ProducerOptions) {
     artifactBase: options.artifactBase,
     htmlPath: matrixHtmlPath,
     logPath: path.join(screenshotCellDir, "logs.txt"),
+    repoRoot: options.repoRoot,
     screenshotPath: matrixScreenshotPath,
   });
 
@@ -575,8 +623,10 @@ export async function runUxMatrixEvidenceProducer(options: ProducerOptions) {
   });
 
   const fixtureProofResult = await captureProducerArtifactFixtureProof({
+    artifactBase: options.artifactBase,
     htmlPath: fixtureHtmlPath,
     logPath: path.join(fixtureProofDir, "logs.txt"),
+    repoRoot: options.repoRoot,
     screenshotPath: path.join(fixtureProofDir, "producer-artifact-fixture.png"),
     skipVisualProof: options.skipVisualProof,
     videoPath: path.join(fixtureProofDir, "producer-artifact-fixture.webm"),
diff --git a/test/scripts/qa-ux-matrix-evidence-producer.test.ts b/test/scripts/qa-ux-matrix-evidence-producer.test.ts
@@ -1,14 +1,18 @@
 // QA UX Matrix evidence producer tests cover operator-facing CLI behavior.
 import { spawnSync } from "node:child_process";
+import fs from "node:fs";
+import os from "node:os";
 import path from "node:path";
 import { describe, expect, it } from "vitest";
 
+const repoRoot = path.resolve(__dirname, "../..");
+
 function runCli(...args: string[]) {
   return spawnSync(
     process.execPath,
     ["--import", "tsx", "scripts/qa/ux-matrix-evidence-producer.ts", ...args],
     {
-      cwd: path.resolve(__dirname, "../.."),
+      cwd: repoRoot,
       encoding: "utf8",
     },
   );
@@ -47,4 +51,31 @@ describe("QA UX Matrix evidence producer CLI", () => {
     expect(result.stderr.trim()).toBe("--artifact-base requires a value");
     expectNoNodeStack(result.stderr);
   });
+
+  it("sanitizes local checkout paths from generated evidence artifacts", () => {
+    const artifactBase = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-ux-evidence-test-"));
+    const fakeRepoRoot = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-ux-repo-test-"));
+    try {
+      const result = runCli(
+        "--artifact-base",
+        artifactBase,
+        "--repo-root",
+        fakeRepoRoot,
+        "--skip-visual-proof",
+      );
+
+      expect(result.status).toBe(0);
+      const evidence = fs.readFileSync(path.join(artifactBase, "qa-evidence.json"), "utf8");
+      const cliLog = fs.readFileSync(
+        path.join(artifactBase, "surfaces", "cli", "stages", "entrypoint-help", "logs.txt"),
+        "utf8",
+      );
+      expect(evidence).not.toContain(fakeRepoRoot);
+      expect(cliLog).not.toContain(fakeRepoRoot);
+      expect(`${evidence}\n${cliLog}`).toContain("<repo-root>");
+    } finally {
+      fs.rmSync(artifactBase, { recursive: true, force: true });
+      fs.rmSync(fakeRepoRoot, { recursive: true, force: true });
+    }
+  });
 });
