qa-lab: support script-backed evidence scenarios (#94276)

Solvely-Colin · RomneyDa · web-flow · commit 591313e80a5e · 2026-06-17T15:09:25.000-07:00
* qa: add script scenario execution kind

* fix(qa-lab): carry suite profile into script producer evidence and simplify artifact path resolution

* fix(qa-lab): keep out-of-repo producer artifacts absolute to avoid ../ traversal refs

---------

Co-authored-by: Dallin Romney &lt;dallinromney@gmail.com&gt;
diff --git a/docs/concepts/qa-e2e-automation.md b/docs/concepts/qa-e2e-automation.md
@@ -942,7 +942,12 @@ Every `qa suite` run writes top-level `qa-evidence.json`,
 `qa-suite-summary.json`, and `qa-suite-report.md` artifacts for the selected
 scenario set. Scenarios that declare `execution.kind: vitest` or
 `execution.kind: playwright` run the matching test path and also write
-per-scenario logs. When `qa suite` is reached through
+per-scenario logs. Scenarios that declare `execution.kind: script` run the
+evidence producer at `execution.path` through `node --import tsx` (with
+`${outputDir}` and `${scenarioId}` expanded in `execution.args`); the producer
+writes its own `qa-evidence.json`, whose entries are imported into the suite
+output and whose artifact paths are resolved relative to that producer
+`qa-evidence.json`. When `qa suite` is reached through
 `qa run --qa-profile`, the same `qa-evidence.json` also includes the profile
 scorecard summary for the selected taxonomy categories.
 Treat it as a discovery aid, not a gate replacement; the selected scenario still needs the right provider mode, live transport, Multipass, Testbox, or release lane for the behavior under test.
diff --git a/extensions/qa-lab/src/coverage-report.test.ts b/extensions/qa-lab/src/coverage-report.test.ts
@@ -67,11 +67,13 @@ function scenarioWithCoverage(params: {
   primary?: readonly string[];
   secondary?: readonly string[];
   sourcePath?: string;
-  executionKind?: "flow" | "vitest" | "playwright";
+  executionKind?: "flow" | "script" | "vitest" | "playwright";
   executionPath?: string;
 }): QaSeedScenarioWithSource {
   const execution =
-    params.executionKind === "vitest" || params.executionKind === "playwright"
+    params.executionKind === "script" ||
+    params.executionKind === "vitest" ||
+    params.executionKind === "playwright"
       ? {
           kind: params.executionKind,
           path: params.executionPath ?? "src/test.test.ts",
@@ -328,6 +330,37 @@ describe("qa coverage report", () => {
     ]);
   });
 
+  it("uses script producer evidence as coverage fulfillment", () => {
+    const report = buildQaScorecardTaxonomyReport({
+      taxonomy: testMaturityTaxonomy({
+        categoryId: TEST_BROWSER_CATEGORY_ID,
+        coverageIds: [TEST_BROWSER_COVERAGE_ID],
+      }),
+      repoRoot: process.cwd(),
+      scenarios: [
+        scenarioWithCoverage({
+          primary: [TEST_BROWSER_COVERAGE_ID],
+          sourcePath: "qa/scenarios/ui/script-evidence-producer.yaml",
+          executionKind: "script",
+          executionPath: "scripts/check-no-conflict-markers.mjs",
+        }),
+      ],
+    });
+
+    expect(report.validationIssues).toStrictEqual([]);
+    expect(report.fulfilledCategoryCount).toBe(1);
+    expect(report.fulfilledFeatureCount).toBe(1);
+    expect(report.categories[0]?.evidence).toStrictEqual([
+      {
+        coverageId: TEST_BROWSER_COVERAGE_ID,
+        kind: "script",
+        path: "scripts/check-no-conflict-markers.mjs",
+        role: "primary",
+        scenarioRefs: ["qa/scenarios/ui/script-evidence-producer.yaml"],
+      },
+    ]);
+  });
+
   it("reports profile membership refs missing from taxonomy categories", () => {
     const report = buildQaScorecardTaxonomyReport({
       taxonomy: testMaturityTaxonomy({
diff --git a/extensions/qa-lab/src/coverage-report.ts b/extensions/qa-lab/src/coverage-report.ts
@@ -465,7 +465,12 @@ function scenarioMatchCommandGroups(matches: readonly QaScenarioSearchMatch[]) {
     group.push(match);
     groups.set(match.executionKind, group);
   }
-  const executionOrder: QaScenarioSearchMatch["executionKind"][] = ["flow", "vitest", "playwright"];
+  const executionOrder: QaScenarioSearchMatch["executionKind"][] = [
+    "flow",
+    "script",
+    "vitest",
+    "playwright",
+  ];
   return executionOrder.flatMap((executionKind) => {
     const group = groups.get(executionKind);
     return group && group.length > 0 ? [{ executionKind, matches: group }] : [];
diff --git a/extensions/qa-lab/src/evidence-summary.ts b/extensions/qa-lab/src/evidence-summary.ts
@@ -349,7 +349,7 @@ function uniqueSortedStrings(values: readonly (string | undefined)[]) {
   );
 }
 
-function resolveQaEvidenceProfile(params: {
+export function resolveQaEvidenceProfile(params: {
   env?: NodeJS.ProcessEnv;
   explicit?: QaEvidenceProfile;
 }) {
@@ -705,6 +705,20 @@ export function buildPlaywrightEvidenceSummary(
   });
 }
 
+export function buildScriptEvidenceSummary(
+  params: QaEvidenceBuildBase & {
+    targets: readonly QaEvidenceTestTargetInput[];
+    results: readonly QaEvidenceTestResultInput[];
+  },
+): QaEvidenceSummaryJson {
+  return buildTestRunnerEvidenceSummary({
+    ...params,
+    defaultRunner: "script",
+    testKind: "script-test",
+    runner: params.runner ?? "script",
+  });
+}
+
 export function buildLiveTransportEvidenceSummary(
   params: QaEvidenceBuildBase & {
     checks: readonly QaEvidenceLiveTransportCheckInput[];
diff --git a/extensions/qa-lab/src/scenario-catalog.ts b/extensions/qa-lab/src/scenario-catalog.ts
@@ -73,6 +73,10 @@ const qaTestFileScenarioExecutionBaseSchema = z.object({
 const qaTestFileScenarioExecutionSchema = z.discriminatedUnion("kind", [
   qaTestFileScenarioExecutionBaseSchema.extend({ kind: z.literal("vitest") }),
   qaTestFileScenarioExecutionBaseSchema.extend({ kind: z.literal("playwright") }),
+  qaTestFileScenarioExecutionBaseSchema.extend({
+    kind: z.literal("script"),
+    args: z.array(z.string()).optional(),
+  }),
 ]);
 
 const qaScenarioExecutionSchema = z.union([
diff --git a/extensions/qa-lab/src/scorecard-taxonomy.ts b/extensions/qa-lab/src/scorecard-taxonomy.ts
@@ -90,7 +90,7 @@ const qaMaturityTaxonomySchema = z
     }
   });
 
-export type QaNativeCoverageEvidenceKind = "vitest" | "playwright";
+export type QaNativeCoverageEvidenceKind = "script" | "vitest" | "playwright";
 export type QaScorecardEvidenceKind = QaNativeCoverageEvidenceKind | "qa-scenario";
 export type QaScorecardEvidenceMode = z.infer<typeof qaScorecardEvidenceModeSchema>;
 type QaCoverageEvidenceRole = z.infer<typeof qaCoverageEvidenceRoleSchema>;
diff --git a/extensions/qa-lab/src/suite-launch.runtime.test.ts b/extensions/qa-lab/src/suite-launch.runtime.test.ts
@@ -74,7 +74,7 @@ describe("qa suite runtime launcher", () => {
     runQaTestFileScenarios.mockImplementation(
       async (params: {
         outputDir: string;
-        scenarios: Array<{ id: string; execution: { kind: "vitest" | "playwright" } }>;
+        scenarios: Array<{ id: string; execution: { kind: "script" | "vitest" | "playwright" } }>;
       }) => {
         const [scenario] = params.scenarios;
         if (!scenario) {
diff --git a/extensions/qa-lab/src/test-file-scenario-runner.test.ts b/extensions/qa-lab/src/test-file-scenario-runner.test.ts
diff --git a/extensions/qa-lab/src/test-file-scenario-runner.ts b/extensions/qa-lab/src/test-file-scenario-runner.ts
diff --git a/qa/scenarios/index.yaml b/qa/scenarios/index.yaml

Original file line number	Diff line number	Diff line change
`@@ -90,7 +90,7 @@ const qaMaturityTaxonomySchema = z`
`90`	`90`	`}`
`91`	`91`	`});`
`92`	`92`
`93`		`-export type QaNativeCoverageEvidenceKind = "vitest" \| "playwright";`
	`93`	`+export type QaNativeCoverageEvidenceKind = "script" \| "vitest" \| "playwright";`
`94`	`94`	`export type QaScorecardEvidenceKind = QaNativeCoverageEvidenceKind \| "qa-scenario";`
`95`	`95`	`export type QaScorecardEvidenceMode = z.infer<typeof qaScorecardEvidenceModeSchema>;`
`96`	`96`	`type QaCoverageEvidenceRole = z.infer<typeof qaCoverageEvidenceRoleSchema>;`