Skip to content

Commit 5361e5a

Browse files
RomneyDaRomneyDa
authored
test(qa): share script evidence writer (#99374)
1 parent 51859fe commit 5361e5a

7 files changed

Lines changed: 587 additions & 207 deletions

File tree

‎extensions/qa-lab/api.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,15 @@ export {
2626
} from "./src/bus-waiters.js";
2727
export { isQaLabCliAvailable, registerQaLabCli } from "./src/cli.js";
2828
export { createQaRunnerRuntime } from "./src/harness-runtime.js";
29+
export {
30+
buildScriptEvidenceSummary,
31+
QA_EVIDENCE_FILENAME,
32+
type QaEvidencePackageSource,
33+
type QaEvidenceStatus,
34+
type QaEvidenceSummaryJson,
35+
validateQaEvidenceSummaryJson,
36+
} from "./src/evidence-summary.js";
37+
export type { QaProviderMode } from "./src/providers/index.js";
2938
export {
3039
type QaLabLatestReport,
3140
type QaLabScenarioOutcome,

‎test/e2e/qa-lab/media/hosted-media-provider-live.test.ts‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,6 @@ describe("hosted media provider live QA producer", () => {
8686
const evidence = buildHostedMediaEvidence({
8787
options,
8888
result: {
89-
artifacts: [{ kind: "log", path: "hosted-media-live.log" }],
9089
durationMs: 10,
9190
status: "pass",
9291
},

‎test/e2e/qa-lab/media/hosted-media-provider-live.ts‎

Lines changed: 80 additions & 102 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,18 @@
11
// Hosted media provider live runner and QA Lab evidence producer.
22
import { spawn } from "node:child_process";
3-
import fs from "node:fs/promises";
43
import path from "node:path";
54
import { pathToFileURL } from "node:url";
65
import {
7-
buildScriptEvidenceSummary,
86
QA_EVIDENCE_FILENAME,
9-
type QaEvidenceStatus,
107
type QaEvidenceSummaryJson,
11-
} from "../../../../extensions/qa-lab/src/evidence-summary.js";
8+
} from "../../../../extensions/qa-lab/api.js";
129
import { spawnPnpmRunner as _spawnPnpmRunner } from "../../../../scripts/pnpm-runner.mjs";
10+
import { createBoundedChildOutput } from "../../../helpers/bounded-child-output.js";
11+
import {
12+
createQaScriptBlockedStatusTracker,
13+
createQaScriptEvidenceWriter,
14+
type QaScriptEvidenceStatus,
15+
} from "../runtime/script-evidence.js";
1316

1417
const SOURCE_PATH = "test/e2e/qa-lab/media/hosted-media-provider-live.ts";
1518
const DEFAULT_PROVIDERS_ENV = "OPENCLAW_QA_HOSTED_MEDIA_PROVIDERS";
@@ -127,10 +130,10 @@ type HostedMediaSuiteDefinition = {
127130
};
128131

129132
type HostedMediaProofResult = {
130-
artifacts: Array<{ kind: string; path: string }>;
133+
artifacts?: Array<{ filePath: string; kind: string }>;
131134
details?: string;
132135
durationMs: number;
133-
status: QaEvidenceStatus;
136+
status: QaScriptEvidenceStatus;
134137
};
135138

136139
const EVIDENCE_SUITES: Record<EvidenceSuiteId, HostedMediaSuiteDefinition> = {
@@ -667,20 +670,6 @@ export function parseHostedMediaOptions(argv: readonly string[]): HostedMediaOpt
667670
};
668671
}
669672

670-
async function writeJson(filePath: string, value: unknown) {
671-
await fs.mkdir(path.dirname(filePath), { recursive: true });
672-
await fs.writeFile(filePath, `${JSON.stringify(value, null, 2)}\n`, "utf8");
673-
}
674-
675-
async function appendText(filePath: string, text: string) {
676-
await fs.mkdir(path.dirname(filePath), { recursive: true });
677-
await fs.appendFile(filePath, text, "utf8");
678-
}
679-
680-
function relativeArtifactPath(options: HostedMediaOptions, filePath: string) {
681-
return path.relative(options.artifactBase, filePath) || path.basename(filePath);
682-
}
683-
684673
function suiteProviderFilter(options: HostedMediaOptions, env: NodeJS.ProcessEnv) {
685674
const suiteEnv = `OPENCLAW_QA_HOSTED_${options.suiteId.toUpperCase()}_PROVIDERS`;
686675
return env[suiteEnv]?.trim() || env[options.providersEnv]?.trim() || "";
@@ -707,30 +696,52 @@ export function buildHostedMediaCommand(params: {
707696
};
708697
}
709698

710-
export function classifyHostedMediaFailureStatus(message: string): QaEvidenceStatus {
711-
const blockedPatterns = [
712-
/no runnable providers matched available auth/i,
713-
/no runnable providers matched the explicit provider selection/i,
714-
/no runnable providers matched explicit provider selection/i,
715-
/no providers with usable auth/i,
716-
];
717-
return blockedPatterns.some((pattern) => pattern.test(message)) ? "blocked" : "fail";
699+
const HOSTED_MEDIA_BLOCKED_PATTERNS = [
700+
/no runnable providers matched available auth/i,
701+
/no runnable providers matched the explicit provider selection/i,
702+
/no runnable providers matched explicit provider selection/i,
703+
/no providers with usable auth/i,
704+
];
705+
706+
export function classifyHostedMediaFailureStatus(message: string): QaScriptEvidenceStatus {
707+
const tracker = createQaScriptBlockedStatusTracker(HOSTED_MEDIA_BLOCKED_PATTERNS);
708+
tracker.append(message);
709+
return tracker.status();
718710
}
719711

720712
function formatCommand(command: string, args: readonly string[]) {
721713
return [command, ...args].map((arg) => JSON.stringify(arg)).join(" ");
722714
}
723715

724-
async function runHostedMediaProof(options: HostedMediaOptions): Promise<HostedMediaProofResult> {
716+
function createHostedMediaEvidenceWriter(options: HostedMediaOptions) {
717+
const definition = EVIDENCE_SUITES[options.suiteId];
718+
return createQaScriptEvidenceWriter({
719+
artifactBase: options.artifactBase,
720+
logFileName: "hosted-media-live.log",
721+
primaryModel: "live-media/hosted-media-provider",
722+
providerMode: "live-frontier",
723+
repoRoot: options.repoRoot,
724+
target: {
725+
id: definition.scenarioId,
726+
title: definition.title,
727+
sourcePath: SOURCE_PATH,
728+
primaryCoverageIds: definition.primaryCoverageIds,
729+
secondaryCoverageIds: definition.secondaryCoverageIds,
730+
docsRefs: definition.docsRefs,
731+
codeRefs: definition.codeRefs,
732+
},
733+
});
734+
}
735+
736+
async function runHostedMediaProof(
737+
options: HostedMediaOptions,
738+
writer: ReturnType<typeof createHostedMediaEvidenceWriter>,
739+
): Promise<HostedMediaProofResult> {
725740
const startedAt = Date.now();
726-
await fs.mkdir(options.artifactBase, { recursive: true });
727-
const logPath = path.join(options.artifactBase, "hosted-media-live.log");
728-
const artifacts = [{ kind: "log", path: relativeArtifactPath(options, logPath) }];
729741
const command = buildHostedMediaCommand({ options });
730742

731-
await appendText(logPath, `$ ${formatCommand(command.command, command.args)}\n`);
732-
await appendText(
733-
logPath,
743+
writer.appendLog(`$ ${formatCommand(command.command, command.args)}\n`);
744+
writer.appendLog(
734745
`suite: ${options.suiteId}\nprovidersEnv: ${options.providersEnv}\nvideoFullModes: ${String(EVIDENCE_SUITES[options.suiteId].videoFullModes === true)}\n`,
735746
);
736747

@@ -740,46 +751,46 @@ async function runHostedMediaProof(options: HostedMediaOptions): Promise<HostedM
740751
env: command.env,
741752
stdio: ["ignore", "pipe", "pipe"],
742753
});
743-
let stdout = "";
744-
let stderr = "";
754+
const stdout = createBoundedChildOutput();
755+
const stderr = createBoundedChildOutput();
756+
const statusTracker = createQaScriptBlockedStatusTracker(HOSTED_MEDIA_BLOCKED_PATTERNS);
745757
child.stdout.setEncoding("utf8");
746758
child.stderr.setEncoding("utf8");
747759
child.stdout.on("data", (chunk: string) => {
748-
stdout += chunk;
760+
stdout.append(chunk);
761+
statusTracker.append(chunk);
749762
});
750763
child.stderr.on("data", (chunk: string) => {
751-
stderr += chunk;
764+
stderr.append(chunk);
765+
statusTracker.append(chunk);
752766
});
753767
child.on("error", reject);
754768
child.on("close", (status, signal) => {
769+
const stdoutText = stdout.text();
770+
const stderrText = stderr.text();
755771
const output = [
756-
stdout ? `\n--- stdout ---\n${stdout}` : "",
757-
stderr ? `\n--- stderr ---\n${stderr}` : "",
772+
stdoutText ? `\n--- stdout ---\n${stdoutText}` : "",
773+
stderrText ? `\n--- stderr ---\n${stderrText}` : "",
758774
].join("");
759-
appendText(logPath, output)
760-
.then(() => {
761-
const durationMs = Math.max(1, Date.now() - startedAt);
762-
if (status === 0 && !signal) {
763-
resolve({
764-
artifacts,
765-
details: `${options.suiteId} hosted media live suite passed`,
766-
durationMs,
767-
status: "pass",
768-
});
769-
return;
770-
}
771-
const details = signal
772-
? `${options.suiteId} hosted media live suite terminated by ${signal}`
773-
: `${options.suiteId} hosted media live suite exited with ${status ?? 1}`;
774-
const combined = `${details}\n${stderr || stdout}`;
775-
resolve({
776-
artifacts,
777-
details: combined,
778-
durationMs,
779-
status: classifyHostedMediaFailureStatus(combined),
780-
});
781-
})
782-
.catch(reject);
775+
writer.appendLog(output);
776+
const durationMs = Math.max(1, Date.now() - startedAt);
777+
if (status === 0 && !signal) {
778+
resolve({
779+
details: `${options.suiteId} hosted media live suite passed`,
780+
durationMs,
781+
status: "pass",
782+
});
783+
return;
784+
}
785+
const details = signal
786+
? `${options.suiteId} hosted media live suite terminated by ${signal}`
787+
: `${options.suiteId} hosted media live suite exited with ${status ?? 1}`;
788+
const combined = `${details}\n${stderrText || stdoutText}`;
789+
resolve({
790+
details: combined,
791+
durationMs,
792+
status: statusTracker.status(),
793+
});
783794
});
784795
});
785796
}
@@ -788,48 +799,15 @@ export function buildHostedMediaEvidence(params: {
788799
options: HostedMediaOptions;
789800
result: HostedMediaProofResult;
790801
}): QaEvidenceSummaryJson {
791-
const definition = EVIDENCE_SUITES[params.options.suiteId];
792-
return buildScriptEvidenceSummary({
793-
artifactPaths: params.result.artifacts,
794-
evidenceMode: "full",
795-
env: process.env,
796-
generatedAt: new Date().toISOString(),
797-
primaryModel: "live-media/hosted-media-provider",
798-
providerMode: "live-frontier",
799-
repoRoot: params.options.repoRoot,
800-
runner: "script",
801-
targets: [
802-
{
803-
id: definition.scenarioId,
804-
title: definition.title,
805-
sourcePath: SOURCE_PATH,
806-
primaryCoverageIds: definition.primaryCoverageIds,
807-
secondaryCoverageIds: definition.secondaryCoverageIds,
808-
docsRefs: definition.docsRefs,
809-
codeRefs: definition.codeRefs,
810-
},
811-
],
812-
results: [
813-
{
814-
id: definition.scenarioId,
815-
status: params.result.status,
816-
durationMs: params.result.durationMs,
817-
failureMessage: params.result.details,
818-
},
819-
],
820-
});
802+
return createHostedMediaEvidenceWriter(params.options).build(params.result);
821803
}
822804

823805
export async function runHostedMediaProviderLiveProducer(
824806
options: HostedMediaOptions,
825807
): Promise<QaEvidenceSummaryJson> {
826-
const result = await runHostedMediaProof(options);
827-
const evidence = buildHostedMediaEvidence({ options, result });
828-
await writeJson(path.join(options.artifactBase, QA_EVIDENCE_FILENAME), evidence);
829-
await writeJson(path.join(options.artifactBase, "latest-run.json"), {
830-
qaEvidence: QA_EVIDENCE_FILENAME,
831-
});
832-
return evidence;
808+
const writer = createHostedMediaEvidenceWriter(options);
809+
const result = await runHostedMediaProof(options, writer);
810+
return await writer.write(result);
833811
}
834812

835813
async function main(argv: string[]) {

‎test/e2e/qa-lab/plugins/clawhub-release-candidate-install.test.ts‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
// ClawHub release candidate producer tests cover blocked script evidence output.
2+
import { spawnSync } from "node:child_process";
3+
import fs from "node:fs/promises";
4+
import os from "node:os";
5+
import path from "node:path";
6+
import { afterEach, describe, expect, it } from "vitest";
7+
import { validateQaEvidenceSummaryJson } from "../../../../extensions/qa-lab/api.js";
8+
9+
const SOURCE_PATH = "test/e2e/qa-lab/plugins/clawhub-release-candidate-install.ts";
10+
const tempRoots: string[] = [];
11+
12+
afterEach(async () => {
13+
await Promise.all(
14+
tempRoots.splice(0).map((root) => fs.rm(root, { recursive: true, force: true })),
15+
);
16+
});
17+
18+
describe("ClawHub release candidate install producer", () => {
19+
it("writes blocked evidence when no candidate tarball is available", async () => {
20+
const artifactBase = await fs.mkdtemp(
21+
path.join(os.tmpdir(), "openclaw-clawhub-release-evidence-"),
22+
);
23+
tempRoots.push(artifactBase);
24+
const missingTarballEnv = "OPENCLAW_TEST_MISSING_RELEASE_CANDIDATE_TARBALL";
25+
const env = { ...process.env };
26+
delete env[missingTarballEnv];
27+
28+
const result = spawnSync(
29+
process.execPath,
30+
[
31+
"--import",
32+
"tsx",
33+
SOURCE_PATH,
34+
"--artifact-base",
35+
artifactBase,
36+
"--tarball-env",
37+
missingTarballEnv,
38+
],
39+
{ cwd: process.cwd(), encoding: "utf8", env },
40+
);
41+
42+
expect(result.status).toBe(0);
43+
const evidence = validateQaEvidenceSummaryJson(
44+
JSON.parse(await fs.readFile(path.join(artifactBase, "qa-evidence.json"), "utf8")),
45+
);
46+
expect(evidence.entries[0]).toMatchObject({
47+
execution: {
48+
artifacts: [{ kind: "log", path: "parallels-npm-update.log", source: "script" }],
49+
},
50+
result: {
51+
status: "blocked",
52+
failure: {
53+
reason: expect.stringContaining(`${missingTarballEnv} is not set`),
54+
},
55+
},
56+
});
57+
expect(result.stdout).toContain("ClawHub release-candidate install status: blocked");
58+
});
59+
});

0 commit comments

Comments
 (0)