Skip to content

Commit 7b03a2a

Browse files
steipetesteipete
authored
test(qa): model non-assistant parity usage (#103650)
1 parent dff4c63 commit 7b03a2a

17 files changed

Lines changed: 337 additions & 14 deletions

‎extensions/qa-lab/src/agentic-parity-report.test.ts‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -962,6 +962,68 @@ status=done`,
962962
expect(report.scenarios[0]?.status).toBe("fail");
963963
});
964964

965+
it("renders explicit non-assistant parity usage as N/A without weakening parity", () => {
966+
const summary = makeRuntimeParitySummary();
967+
summary.run = {
968+
...summary.run,
969+
providerMode: "live-frontier",
970+
};
971+
const scenario = summary.scenarios[0];
972+
if (!scenario?.runtimeParity) {
973+
throw new Error("runtime parity fixture missing");
974+
}
975+
scenario.runtimeParity.runtimeParityUsage = {
976+
expectation: "not-applicable",
977+
reason: "Local fixture only; no assistant turn runs.",
978+
};
979+
scenario.runtimeParity.cells.openclaw.usage = {
980+
inputTokens: 0,
981+
outputTokens: 0,
982+
totalTokens: 0,
983+
};
984+
scenario.runtimeParity.cells.codex.usage = {
985+
inputTokens: 0,
986+
outputTokens: 0,
987+
totalTokens: 0,
988+
};
989+
990+
const report = buildQaRuntimeParityReport({ summary });
991+
const markdown = renderQaRuntimeParityMarkdownReport(report);
992+
993+
expect(report.pass).toBe(true);
994+
expect(report.failures).toEqual([]);
995+
expect(report.scenarios[0]?.status).toBe("pass");
996+
expect(markdown).toContain("- openclaw: pass (1 tool calls, N/A tokens)");
997+
expect(markdown).toContain(
998+
"- assistant-message usage: N/A (Local fixture only; no assistant turn runs.)",
999+
);
1000+
});
1001+
1002+
it("keeps non-assistant parity runtime failures red", () => {
1003+
const summary = makeRuntimeParitySummary();
1004+
summary.run = {
1005+
...summary.run,
1006+
providerMode: "live-frontier",
1007+
};
1008+
const scenario = summary.scenarios[0];
1009+
if (!scenario?.runtimeParity) {
1010+
throw new Error("runtime parity fixture missing");
1011+
}
1012+
scenario.runtimeParity.runtimeParityUsage = {
1013+
expectation: "not-applicable",
1014+
reason: "Local fixture only; no assistant turn runs.",
1015+
};
1016+
scenario.runtimeParity.cells.openclaw.usage.totalTokens = 0;
1017+
scenario.runtimeParity.cells.codex.usage.totalTokens = 0;
1018+
scenario.runtimeParity.cells.codex.runtimeErrorClass = "auth";
1019+
1020+
const report = buildQaRuntimeParityReport({ summary });
1021+
1022+
expect(report.pass).toBe(false);
1023+
expect(report.failedScenarios).toBe(1);
1024+
expect(report.failures).toContain("Approval turn tool followthrough drift=none.");
1025+
});
1026+
9651027
it("fails runtime parity reports with no executed scenarios", () => {
9661028
const report = buildQaRuntimeParityReport({
9671029
summary: {

‎extensions/qa-lab/src/agentic-parity-report.ts‎

Lines changed: 27 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,17 @@ import {
33
QA_AGENTIC_PARITY_SCENARIO_TITLES,
44
QA_AGENTIC_PARITY_TOOL_BACKED_SCENARIO_TITLES,
55
} from "./agentic-parity.js";
6-
import type { RuntimeId, RuntimeParityDrift, RuntimeParityResult } from "./runtime-parity.js";
7-
import { isRuntimeParityResultPass, runtimeParityCellStatus } from "./runtime-parity.js";
6+
import type {
7+
RuntimeId,
8+
RuntimeParityDrift,
9+
RuntimeParityResult,
10+
RuntimeParityUsagePolicy,
11+
} from "./runtime-parity.js";
12+
import {
13+
isRuntimeParityResultPass,
14+
resolveRuntimeParityUsagePolicy,
15+
runtimeParityCellStatus,
16+
} from "./runtime-parity.js";
817

918
type QaParityReportStep = {
1019
name: string;
@@ -57,6 +66,7 @@ export type QaRuntimeParitySuiteSummary = Omit<QaParitySuiteSummary, "scenarios"
5766
type QaRuntimeParityScenarioReport = {
5867
name: string;
5968
status: "pass" | "fail";
69+
runtimeParityUsage: RuntimeParityUsagePolicy;
6070
drift: RuntimeParityDrift | "missing";
6171
driftDetails?: string;
6272
openclawStatus: "pass" | "fail" | "missing";
@@ -651,6 +661,7 @@ export function buildQaRuntimeParityReport(params: {
651661
return {
652662
name: scenario.name,
653663
status: scenario.status === "pass" ? "pass" : "fail",
664+
runtimeParityUsage: resolveRuntimeParityUsagePolicy(undefined),
654665
drift: "missing",
655666
driftDetails: scenario.details,
656667
openclawStatus: "missing",
@@ -667,9 +678,11 @@ export function buildQaRuntimeParityReport(params: {
667678
const openclawStatus = runtimeParityCellStatus(openclawCell);
668679
const codexStatus = runtimeParityCellStatus(codexCell);
669680
const parityStatus = isRuntimeParityResultPass(parity) ? "pass" : "fail";
681+
const runtimeParityUsage = resolveRuntimeParityUsagePolicy(parity.runtimeParityUsage);
670682
const reportScenario = {
671683
name: scenario.name,
672684
status: parityStatus,
685+
runtimeParityUsage,
673686
drift: parity.drift,
674687
driftDetails: parity.driftDetails,
675688
openclawStatus,
@@ -684,9 +697,10 @@ export function buildQaRuntimeParityReport(params: {
684697
`${scenario.name} drift=${parity.drift}${parity.driftDetails ? ` (${parity.driftDetails})` : ""}.`,
685698
);
686699
}
687-
const usageFailure = requiresLiveUsage
688-
? describeLiveUsageFailure(scenario.name, reportScenario)
689-
: undefined;
700+
const usageFailure =
701+
requiresLiveUsage && runtimeParityUsage.expectation === "assistant-message-required"
702+
? describeLiveUsageFailure(scenario.name, reportScenario)
703+
: undefined;
690704
if (usageFailure) {
691705
failures.push(usageFailure);
692706
return { ...reportScenario, status: "fail" };
@@ -755,15 +769,21 @@ export function renderQaRuntimeParityMarkdownReport(report: QaRuntimeParityRepor
755769

756770
lines.push("## Scenario Comparison", "");
757771
for (const scenario of report.scenarios) {
772+
const usageNotApplicable = scenario.runtimeParityUsage.expectation === "not-applicable";
773+
const openclawTokens = usageNotApplicable ? "N/A" : String(scenario.openclawTokens);
774+
const codexTokens = usageNotApplicable ? "N/A" : String(scenario.codexTokens);
758775
lines.push(`### ${scenario.name}`, "");
759776
lines.push(`- status: ${scenario.status}`);
760777
lines.push(`- drift: ${scenario.drift}`);
761778
lines.push(
762-
`- openclaw: ${scenario.openclawStatus} (${scenario.openclawToolCalls} tool calls, ${scenario.openclawTokens} tokens)`,
779+
`- openclaw: ${scenario.openclawStatus} (${scenario.openclawToolCalls} tool calls, ${openclawTokens} tokens)`,
763780
);
764781
lines.push(
765-
`- codex: ${scenario.codexStatus} (${scenario.codexToolCalls} tool calls, ${scenario.codexTokens} tokens)`,
782+
`- codex: ${scenario.codexStatus} (${scenario.codexToolCalls} tool calls, ${codexTokens} tokens)`,
766783
);
784+
if (scenario.runtimeParityUsage.expectation === "not-applicable") {
785+
lines.push(`- assistant-message usage: N/A (${scenario.runtimeParityUsage.reason})`);
786+
}
767787
if (scenario.driftDetails) {
768788
lines.push(`- details: ${scenario.driftDetails}`);
769789
}

‎extensions/qa-lab/src/runtime-parity.test.ts‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import {
44
__testing,
55
captureRuntimeParityCell,
66
isRuntimeParityResultPass,
7+
resolveRuntimeParityUsagePolicy,
78
runRuntimeParityScenario,
89
type RuntimeId,
910
type RuntimeParityCell,
@@ -100,6 +101,37 @@ describe("runtime parity", () => {
100101
});
101102

102103
expect(result.drift).toBe("none");
104+
expect(result.runtimeParityUsage).toEqual({
105+
expectation: "assistant-message-required",
106+
});
107+
});
108+
109+
it("preserves explicit usage-not-applicable metadata on parity results", async () => {
110+
const result = await runRuntimeParityScenario({
111+
scenarioId: "local-fixture",
112+
runtimeParityUsage: {
113+
expectation: "not-applicable",
114+
reason: " Local fixture only; no assistant turn runs. ",
115+
},
116+
runCell: async (runtime) => ({
117+
scenarioStatus: "pass",
118+
cell: makeRuntimeParityCell(runtime, []),
119+
}),
120+
});
121+
122+
expect(result.runtimeParityUsage).toEqual({
123+
expectation: "not-applicable",
124+
reason: "Local fixture only; no assistant turn runs.",
125+
});
126+
});
127+
128+
it("defaults malformed usage metadata to assistant-message-required", () => {
129+
expect(resolveRuntimeParityUsagePolicy({ expectation: "not-applicable" })).toEqual({
130+
expectation: "assistant-message-required",
131+
});
132+
expect(
133+
resolveRuntimeParityUsagePolicy({ expectation: "not-applicable", reason: " " }),
134+
).toEqual({ expectation: "assistant-message-required" });
103135
});
104136

105137
it("classifies planned-only matching tool calls as failure-mode", async () => {

‎extensions/qa-lab/src/runtime-parity.ts‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,10 @@ export type RuntimeParityUsage = {
3131
cacheWrite?: number;
3232
};
3333

34+
export type RuntimeParityUsagePolicy =
35+
| { expectation: "assistant-message-required" }
36+
| { expectation: "not-applicable"; reason: string };
37+
3438
export type RuntimeParityCell = {
3539
runtime: RuntimeId;
3640
transcriptBytes: string;
@@ -54,6 +58,7 @@ export type RuntimeParityDrift =
5458

5559
export type RuntimeParityResult = {
5660
scenarioId: string;
61+
runtimeParityUsage?: RuntimeParityUsagePolicy;
5762
cells: {
5863
openclaw: RuntimeParityCell;
5964
codex: RuntimeParityCell;
@@ -62,6 +67,22 @@ export type RuntimeParityResult = {
6267
driftDetails?: string;
6368
};
6469

70+
export function resolveRuntimeParityUsagePolicy(value: unknown): RuntimeParityUsagePolicy {
71+
// Legacy or malformed summaries must not silently disable live-usage proof.
72+
if (!value || typeof value !== "object") {
73+
return { expectation: "assistant-message-required" };
74+
}
75+
const candidate = value as { expectation?: unknown; reason?: unknown };
76+
if (
77+
candidate.expectation === "not-applicable" &&
78+
typeof candidate.reason === "string" &&
79+
candidate.reason.trim()
80+
) {
81+
return { expectation: "not-applicable", reason: candidate.reason.trim() };
82+
}
83+
return { expectation: "assistant-message-required" };
84+
}
85+
6586
export type RuntimeParityScenarioExecution = {
6687
scenarioStatus: "pass" | "fail";
6788
scenarioDetails?: string;
@@ -1065,6 +1086,7 @@ export async function captureRuntimeParityCell(
10651086

10661087
export async function runRuntimeParityScenario(params: {
10671088
scenarioId: string;
1089+
runtimeParityUsage?: RuntimeParityUsagePolicy;
10681090
runCell: (runtime: RuntimeId) => Promise<RuntimeParityScenarioExecution>;
10691091
}): Promise<RuntimeParityResult> {
10701092
const openclaw = await params.runCell("openclaw");
@@ -1077,6 +1099,7 @@ export async function runRuntimeParityScenario(params: {
10771099
});
10781100
return {
10791101
scenarioId: params.scenarioId,
1102+
runtimeParityUsage: resolveRuntimeParityUsagePolicy(params.runtimeParityUsage),
10801103
cells: {
10811104
openclaw: openclaw.cell,
10821105
codex: codex.cell,

‎extensions/qa-lab/src/scenario-catalog.test.ts‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -305,6 +305,37 @@ describe("qa scenario catalog", () => {
305305
expect(readQaScenarioExecutionConfig(soak.id)).toMatchObject({ turnCount: 100 });
306306
});
307307

308+
it("marks only non-assistant runtime parity fixtures as usage not applicable", () => {
309+
const notApplicable = readQaScenarioPack()
310+
.scenarios.filter((scenario) => scenario.runtimeParityUsage?.expectation === "not-applicable")
311+
.map((scenario) => scenario.id)
312+
.toSorted();
313+
314+
expect(notApplicable).toStrictEqual(
315+
[
316+
"auth-profile-codex-mixed-profiles",
317+
"auth-profile-doctor-migration-safety",
318+
"codex-plugin-cold-install",
319+
"codex-plugin-install-race",
320+
"codex-plugin-pinned-new",
321+
"codex-plugin-pinned-old",
322+
"plugin-manifest-contract-health",
323+
].toSorted(),
324+
);
325+
for (const scenarioId of notApplicable) {
326+
const scenario = readQaScenarioById(scenarioId);
327+
expect(scenario.runtimeParityTier).toBeDefined();
328+
expect(scenario.runtimeParityUsage).toMatchObject({
329+
expectation: "not-applicable",
330+
});
331+
if (scenario.runtimeParityUsage?.expectation === "not-applicable") {
332+
expect(scenario.runtimeParityUsage.reason).toContain("no assistant turn runs");
333+
}
334+
}
335+
expect(readQaScenarioById("runtime-tool-fs-read").runtimeParityUsage).toBeUndefined();
336+
expect(readQaScenarioById("plugin-hook-health-sentinel").runtimeParityUsage).toBeUndefined();
337+
});
338+
308339
it("loads runtime tool fixture metadata for standard and optional lanes", () => {
309340
const applyPatch = readQaScenarioById("runtime-tool-apply-patch");
310341
const messageTool = readQaScenarioById("runtime-tool-message-tool");

‎extensions/qa-lab/src/scenario-catalog.ts‎

Lines changed: 26 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,15 @@ const qaScenarioGatewayRuntimeSchema = z.object({
161161

162162
export const QA_RUNTIME_PARITY_TIERS = ["standard", "optional", "live-only", "soak"] as const;
163163
const qaRuntimeParityTierSchema = z.enum(QA_RUNTIME_PARITY_TIERS);
164+
const qaRuntimeParityUsageSchema = z.discriminatedUnion("expectation", [
165+
z.object({
166+
expectation: z.literal("assistant-message-required"),
167+
}),
168+
z.object({
169+
expectation: z.literal("not-applicable"),
170+
reason: z.string().trim().min(1),
171+
}),
172+
]);
164173

165174
const qaFlowCallActionSchema = z.object({
166175
call: z.string().trim().min(1),
@@ -271,6 +280,7 @@ const qaSeedScenarioBodySchema = z.object({
271280
surface: z.string().trim().min(1),
272281
category: z.string().trim().min(1).optional(),
273282
runtimeParityTier: qaRuntimeParityTierSchema.optional(),
283+
runtimeParityUsage: qaRuntimeParityUsageSchema.optional(),
274284
coverage: qaScenarioCoverageSchema.optional(),
275285
surfaces: z.array(z.string().trim().min(1)).min(1).optional(),
276286
risk: z.enum(["low", "medium", "high"]).optional(),
@@ -292,11 +302,21 @@ const qaSeedScenarioSchema = qaSeedScenarioBodySchema.extend({
292302
title: z.string().trim().min(1),
293303
});
294304

295-
const qaScenarioFileSchema = z.object({
296-
title: z.string().trim().min(1),
297-
scenario: qaSeedScenarioBodySchema,
298-
flow: qaFlowSchema.optional(),
299-
});
305+
const qaScenarioFileSchema = z
306+
.object({
307+
title: z.string().trim().min(1),
308+
scenario: qaSeedScenarioBodySchema,
309+
flow: qaFlowSchema.optional(),
310+
})
311+
.superRefine((file, ctx) => {
312+
if (file.scenario.runtimeParityUsage && !file.scenario.runtimeParityTier) {
313+
ctx.addIssue({
314+
code: z.ZodIssueCode.custom,
315+
path: ["scenario", "runtimeParityUsage"],
316+
message: "runtimeParityUsage requires runtimeParityTier",
317+
});
318+
}
319+
});
300320

301321
const qaScenarioPackSchema = z.object({
302322
version: z.number().int().positive(),
@@ -318,6 +338,7 @@ const qaScenarioPackFileSchema = z.object({
318338
export type QaScenarioExecution = z.infer<typeof qaScenarioExecutionSchema>;
319339
export type QaScenarioFlow = z.infer<typeof qaFlowSchema>;
320340
export type QaRuntimeParityTier = z.infer<typeof qaRuntimeParityTierSchema>;
341+
export type QaRuntimeParityUsage = z.infer<typeof qaRuntimeParityUsageSchema>;
321342
export type QaSeedScenario = z.infer<typeof qaSeedScenarioSchema>;
322343
export type QaSeedScenarioWithSource = QaSeedScenario & {
323344
sourcePath: string;

‎extensions/qa-lab/src/suite.summary-json.test.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -175,6 +175,10 @@ describe("buildQaSuiteSummaryJson", () => {
175175
steps: [],
176176
runtimeParity: {
177177
scenarioId: "scenario-a",
178+
runtimeParityUsage: {
179+
expectation: "not-applicable" as const,
180+
reason: "Local fixture only; no assistant turn runs.",
181+
},
178182
drift: "none" as const,
179183
cells: {
180184
openclaw: {
@@ -204,6 +208,10 @@ describe("buildQaSuiteSummaryJson", () => {
204208
expect(json.scenarios[0]).toMatchObject({
205209
runtimeParity: {
206210
scenarioId: "scenario-a",
211+
runtimeParityUsage: {
212+
expectation: "not-applicable",
213+
reason: "Local fixture only; no assistant turn runs.",
214+
},
207215
drift: "none",
208216
},
209217
});

‎extensions/qa-lab/src/suite.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -846,6 +846,7 @@ async function runQaRuntimeParitySuite(params: {
846846

847847
const parity = await runRuntimeParityScenario({
848848
scenarioId: scenario.id,
849+
runtimeParityUsage: scenario.runtimeParityUsage,
849850
runCell: async (runtime) => {
850851
const cellOutputDir = path.join(
851852
params.outputDir,

0 commit comments

Comments
 (0)