Skip to content

Commit c79fb2c

Browse files
vincentkocvincentkoc
committed
fix(qa): record checked-out ref in evidence
1 parent c588606 commit c79fb2c

9 files changed

Lines changed: 92 additions & 13 deletions

File tree

‎extensions/qa-lab/src/evidence-summary.test.ts‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
// Qa Lab tests cover QA evidence summary behavior.
2+
import { execFileSync } from "node:child_process";
23
import { describe, expect, it } from "vitest";
34
import {
45
QA_EVIDENCE_SUMMARY_KIND,
@@ -123,6 +124,29 @@ describe("evidence summary", () => {
123124
});
124125
});
125126

127+
it("prefers the checked-out ref over an inherited GitHub event SHA", () => {
128+
const repoRoot = process.cwd();
129+
const checkedOutRef = execFileSync("git", ["rev-parse", "--verify", "HEAD"], {
130+
cwd: repoRoot,
131+
encoding: "utf8",
132+
}).trim();
133+
const evidence = buildQaSuiteEvidenceSummary({
134+
artifactPaths: [],
135+
channelId: "qa-channel",
136+
env: {
137+
GITHUB_SHA: "bd479958c04a1eadbda8b6105e0722588d71e9ad",
138+
} as NodeJS.ProcessEnv,
139+
generatedAt: "2026-06-24T12:00:00.000Z",
140+
primaryModel: "mock-openai/gpt-5.5",
141+
providerMode: "mock-openai",
142+
repoRoot,
143+
scenarioDefinitions: [{ id: "ref-probe", title: "Ref probe" }],
144+
scenarioResults: [{ name: "Ref probe", status: "pass" }],
145+
});
146+
147+
expect(evidence.entries[0]?.execution?.environment.ref).toBe(checkedOutRef);
148+
});
149+
126150
it("builds Telegram live transport evidence entries", () => {
127151
const evidence = buildLiveTransportEvidenceSummary({
128152
artifactPaths: [

‎extensions/qa-lab/src/evidence-summary.ts‎

Lines changed: 38 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
// Qa Lab plugin module implements QA evidence summary behavior.
2+
import { execFileSync } from "node:child_process";
23
import { z } from "zod";
34
import { splitQaModelRef } from "./model-selection.js";
45
import { getQaProvider, type QaProviderMode } from "./providers/index.js";
@@ -288,6 +289,7 @@ type QaEvidenceBuildBase = {
288289
channelDriver?: string;
289290
packageSource?: QaEvidencePackageSource;
290291
profile?: QaEvidenceProfile;
292+
repoRoot?: string;
291293
runner?: string;
292294
};
293295

@@ -388,9 +390,31 @@ function resolveQaEvidenceChannelDriver(params: { env?: NodeJS.ProcessEnv; fallb
388390
return id ? { id } : undefined;
389391
}
390392

391-
function resolveQaEvidenceEnvironment(env: NodeJS.ProcessEnv | undefined) {
393+
function resolveQaEvidenceCheckoutRef(repoRoot?: string) {
394+
try {
395+
const ref = execFileSync("git", ["rev-parse", "--verify", "HEAD"], {
396+
cwd: repoRoot ?? process.cwd(),
397+
encoding: "utf8",
398+
stdio: ["ignore", "pipe", "ignore"],
399+
}).trim();
400+
return ref || undefined;
401+
} catch {
402+
return undefined;
403+
}
404+
}
405+
406+
export function resolveQaEvidenceEnvironment(params: {
407+
env?: NodeJS.ProcessEnv;
408+
repoRoot?: string;
409+
}) {
392410
return {
393-
ref: env?.OPENCLAW_QA_REF?.trim() || env?.GITHUB_SHA?.trim() || null,
411+
// GitHub's GITHUB_SHA describes the workflow event, not necessarily the
412+
// checked-out ref selected by a manual or remote QA run.
413+
ref:
414+
params.env?.OPENCLAW_QA_REF?.trim() ||
415+
resolveQaEvidenceCheckoutRef(params.repoRoot) ||
416+
params.env?.GITHUB_SHA?.trim() ||
417+
null,
394418
os: process.platform,
395419
nodeVersion: process.version,
396420
};
@@ -550,7 +574,10 @@ export function buildQaSuiteEvidenceSummary(
550574
},
551575
): QaEvidenceSummaryJson {
552576
const provider = buildQaEvidenceProvider(params);
553-
const environment = resolveQaEvidenceEnvironment(params.env);
577+
const environment = resolveQaEvidenceEnvironment({
578+
env: params.env,
579+
repoRoot: params.repoRoot,
580+
});
554581
const packageSource = resolveQaEvidenceBuildPackageSource(params);
555582
const runner = resolveQaEvidenceRunner({ env: params.env, fallback: params.runner });
556583
const profile = resolveQaEvidenceProfile({
@@ -622,7 +649,10 @@ function buildTestRunnerEvidenceSummary(
622649
},
623650
): QaEvidenceSummaryJson {
624651
const provider = buildQaEvidenceProvider(params);
625-
const environment = resolveQaEvidenceEnvironment(params.env);
652+
const environment = resolveQaEvidenceEnvironment({
653+
env: params.env,
654+
repoRoot: params.repoRoot,
655+
});
626656
const packageSource = resolveQaEvidenceBuildPackageSource(params);
627657
const runner = resolveQaEvidenceRunner({
628658
env: params.env,
@@ -726,7 +756,10 @@ export function buildLiveTransportEvidenceSummary(
726756
},
727757
): QaEvidenceSummaryJson {
728758
const provider = buildQaEvidenceProvider(params);
729-
const environment = resolveQaEvidenceEnvironment(params.env);
759+
const environment = resolveQaEvidenceEnvironment({
760+
env: params.env,
761+
repoRoot: params.repoRoot,
762+
});
730763
const packageSource = resolveQaEvidenceBuildPackageSource(params);
731764
const runner = resolveQaEvidenceRunner({ env: params.env, fallback: params.runner });
732765
const profile = resolveQaEvidenceProfile({

‎extensions/qa-lab/src/live-transports/discord/discord-live.runtime.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1863,6 +1863,7 @@ export async function runDiscordQaLive(params: {
18631863
generatedAt: finishedAt,
18641864
primaryModel,
18651865
providerMode,
1866+
repoRoot,
18661867
transportId: "discord",
18671868
});
18681869
await fs.writeFile(

‎extensions/qa-lab/src/live-transports/slack/slack-live.runtime.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2037,6 +2037,7 @@ export async function runSlackQaLive(params: {
20372037
generatedAt: finishedAt,
20382038
primaryModel,
20392039
providerMode,
2040+
repoRoot,
20402041
transportId: "slack",
20412042
});
20422043
await fs.writeFile(

‎extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2188,6 +2188,7 @@ export async function runTelegramQaLive(params: {
21882188
generatedAt: finishedAt,
21892189
primaryModel,
21902190
providerMode,
2191+
repoRoot,
21912192
checks: scenarioResults,
21922193
transportId: "telegram",
21932194
});

‎extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3282,6 +3282,7 @@ export async function runWhatsAppQaLive(params: {
32823282
generatedAt: finishedAt,
32833283
primaryModel,
32843284
providerMode,
3285+
repoRoot,
32853286
transportId: "whatsapp",
32863287
});
32873288
await fs.writeFile(

‎extensions/qa-lab/src/suite.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -848,6 +848,7 @@ async function runQaRuntimeParitySuite(params: {
848848
const finishedAt = new Date();
849849
const { evidence, evidencePath, report, reportPath, summaryPath } = await writeQaSuiteArtifacts(
850850
{
851+
repoRoot: params.repoRoot,
851852
outputDir: params.outputDir,
852853
startedAt: params.startedAt,
853854
finishedAt,
@@ -900,6 +901,7 @@ async function runQaRuntimeParitySuite(params: {
900901
}
901902

902903
async function writeQaSuiteArtifacts(params: {
904+
repoRoot?: string;
903905
outputDir: string;
904906
startedAt: Date;
905907
finishedAt: Date;
@@ -974,6 +976,7 @@ async function writeQaSuiteArtifacts(params: {
974976
generatedAt: params.finishedAt.toISOString(),
975977
primaryModel: params.primaryModel,
976978
providerMode: params.providerMode,
979+
repoRoot: params.repoRoot,
977980
scenarioDefinitions: params.scenarioDefinitions,
978981
scenarioResults: params.scenarios,
979982
})
@@ -1296,6 +1299,7 @@ export async function runQaFlowSuite(params?: QaSuiteRunParams): Promise<QaSuite
12961299
.then(async () => {
12971300
const partialFinishedAt = new Date();
12981301
const { report, reportPath } = await writeQaSuiteArtifacts({
1302+
repoRoot,
12991303
outputDir,
13001304
startedAt,
13011305
finishedAt: partialFinishedAt,
@@ -1448,6 +1452,7 @@ export async function runQaFlowSuite(params?: QaSuiteRunParams): Promise<QaSuite
14481452
});
14491453
const { evidence, evidencePath, report, reportPath, summaryPath } =
14501454
await writeQaSuiteArtifacts({
1455+
repoRoot,
14511456
outputDir,
14521457
startedAt,
14531458
finishedAt,
@@ -1720,6 +1725,7 @@ export async function runQaFlowSuite(params?: QaSuiteRunParams): Promise<QaSuite
17201725
});
17211726
const { evidence, evidencePath, report, reportPath, summaryPath } = await writeQaSuiteArtifacts(
17221727
{
1728+
repoRoot,
17231729
outputDir,
17241730
startedAt,
17251731
finishedAt,

‎extensions/qa-lab/src/test-file-scenario-runner.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -555,6 +555,7 @@ function buildTestFileEvidence(params: {
555555
kind: QaTestFileExecutionKind;
556556
primaryModel: string;
557557
providerMode: QaProviderMode;
558+
repoRoot: string;
558559
results: readonly QaTestFileScenarioResult[];
559560
evidenceMode?: QaScorecardEvidenceMode;
560561
env?: NodeJS.ProcessEnv;
@@ -581,6 +582,7 @@ function buildTestFileEvidence(params: {
581582
generatedAt: params.generatedAt,
582583
primaryModel: params.primaryModel,
583584
providerMode: params.providerMode,
585+
repoRoot: params.repoRoot,
584586
targets: fallbackResults.map((result) => buildScenarioEvidenceTarget(result.scenario)),
585587
results: fallbackResults.map((result) => ({
586588
id: result.scenario.id,
@@ -616,6 +618,7 @@ function buildTestFileEvidence(params: {
616618
generatedAt: params.generatedAt,
617619
primaryModel: params.primaryModel,
618620
providerMode: params.providerMode,
621+
repoRoot: params.repoRoot,
619622
targets: params.results.map((result) => buildScenarioEvidenceTarget(result.scenario)),
620623
results: params.results.map((result) => ({
621624
id: result.scenario.id,
@@ -802,6 +805,7 @@ export async function runQaTestFileScenarios(
802805
kind,
803806
primaryModel: params.primaryModel,
804807
providerMode: params.providerMode,
808+
repoRoot: params.repoRoot,
805809
results,
806810
});
807811
const paths = await writeTestFileEvidenceFile({

‎scripts/qa/ux-matrix-evidence-producer.ts‎

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import {
1010
QA_EVIDENCE_FILENAME,
1111
QA_EVIDENCE_SUMMARY_KIND,
1212
QA_EVIDENCE_SUMMARY_SCHEMA_VERSION,
13+
resolveQaEvidenceEnvironment,
1314
validateQaEvidenceSummaryJson,
1415
type QaEvidenceStatus,
1516
type QaEvidenceSummaryEntry,
@@ -174,15 +175,15 @@ function sanitizeArtifactText(
174175

175176
function buildExecution(params: {
176177
artifacts: MatrixCell["artifacts"];
178+
repoRoot: string;
177179
source: string;
178180
}): QaEvidenceSummaryEntry["execution"] {
179181
return {
180182
runner: "ux-matrix-script-producer",
181-
environment: {
182-
ref: process.env.OPENCLAW_QA_REF?.trim() || process.env.GITHUB_SHA?.trim() || null,
183-
os: process.platform,
184-
nodeVersion: process.version,
185-
},
183+
environment: resolveQaEvidenceEnvironment({
184+
env: process.env,
185+
repoRoot: params.repoRoot,
186+
}),
186187
provider: {
187188
id: "ux-matrix",
188189
live: false,
@@ -202,7 +203,7 @@ function buildExecution(params: {
202203
};
203204
}
204205

205-
function buildEvidenceEntry(cell: MatrixCell): QaEvidenceSummaryEntry {
206+
function buildEvidenceEntry(cell: MatrixCell, repoRoot: string): QaEvidenceSummaryEntry {
206207
const source = `ux-matrix:${cell.surface}:${cell.stage}`;
207208
return {
208209
test: {
@@ -221,6 +222,7 @@ function buildEvidenceEntry(cell: MatrixCell): QaEvidenceSummaryEntry {
221222
],
222223
execution: buildExecution({
223224
artifacts: cell.artifacts,
225+
repoRoot,
224226
source,
225227
}),
226228
result: {
@@ -243,13 +245,14 @@ function buildEvidenceEntry(cell: MatrixCell): QaEvidenceSummaryEntry {
243245
function buildEvidenceSummary(params: {
244246
cells: readonly MatrixCell[];
245247
generatedAt: string;
248+
repoRoot: string;
246249
}): QaEvidenceSummaryJson {
247250
return validateQaEvidenceSummaryJson({
248251
kind: QA_EVIDENCE_SUMMARY_KIND,
249252
schemaVersion: QA_EVIDENCE_SUMMARY_SCHEMA_VERSION,
250253
generatedAt: params.generatedAt,
251254
evidenceMode: "full",
252-
entries: params.cells.map(buildEvidenceEntry),
255+
entries: params.cells.map((cell) => buildEvidenceEntry(cell, params.repoRoot)),
253256
});
254257
}
255258

@@ -693,6 +696,7 @@ export async function runUxMatrixEvidenceProducer(options: ProducerOptions) {
693696
const previewEvidence = buildEvidenceSummary({
694697
cells: initialCells,
695698
generatedAt: new Date().toISOString(),
699+
repoRoot: options.repoRoot,
696700
});
697701
const screenshotLog = await fs.readFile(path.join(screenshotCellDir, "logs.txt"), "utf8");
698702
await writeProducerArtifactFixtureHtml({
@@ -753,7 +757,11 @@ export async function runUxMatrixEvidenceProducer(options: ProducerOptions) {
753757
...initialCells,
754758
];
755759

756-
const evidence = buildEvidenceSummary({ cells, generatedAt: new Date().toISOString() });
760+
const evidence = buildEvidenceSummary({
761+
cells,
762+
generatedAt: new Date().toISOString(),
763+
repoRoot: options.repoRoot,
764+
});
757765
await writeProducerArtifactFixtureHtml({
758766
artifactBase: options.artifactBase,
759767
evidence,

0 commit comments

Comments
 (0)