Skip to content

Commit 0c65719

Browse files
vincentkocvincentkoc
committed
fix(qa): fail runtime parity on cell failures
1 parent 6ffa0fb commit 0c65719

4 files changed

Lines changed: 163 additions & 19 deletions

File tree

‎extensions/qa-lab/src/runtime-parity.test.ts‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,28 @@ describe("runtime parity", () => {
114114
expect(isRuntimeParityResultPass(result)).toBe(true);
115115
});
116116

117+
it("does not mask runtime cell scenario failures behind drift", async () => {
118+
const result = await runRuntimeParityScenario({
119+
scenarioId: "failed-cell-with-drift",
120+
runCell: async (runtime) => ({
121+
scenarioStatus: runtime === "codex" ? "fail" : "pass",
122+
cell: makeRuntimeParityCell(runtime, [
123+
{
124+
tool: "web_search",
125+
argsHash: "same-args",
126+
resultHash: runtime === "codex" ? "failed-result" : "ok-result",
127+
},
128+
]),
129+
}),
130+
});
131+
132+
expect(result).toMatchObject({
133+
drift: "failure-mode",
134+
driftDetails: "scenario status differs (pass vs fail)",
135+
});
136+
expect(isRuntimeParityResultPass(result)).toBe(false);
137+
});
138+
117139
it("prefers transcript tool results when mock debug rows are incomplete", () => {
118140
const resolved = __testing.resolveRuntimeParityToolCalls({
119141
mockToolCalls: [

‎extensions/qa-lab/src/runtime-parity.ts‎

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -854,6 +854,21 @@ function classifyRuntimeParityCells(params: {
854854
};
855855
}
856856

857+
if (
858+
params.openclawScenarioStatus === "fail" ||
859+
params.codexScenarioStatus === "fail" ||
860+
!isRuntimeParityCellPassable(params.openclaw) ||
861+
!isRuntimeParityCellPassable(params.codex)
862+
) {
863+
return {
864+
drift: "failure-mode",
865+
driftDetails:
866+
params.openclawScenarioStatus === params.codexScenarioStatus
867+
? "at least one runtime failed"
868+
: `scenario status differs (${params.openclawScenarioStatus} vs ${params.codexScenarioStatus})`,
869+
};
870+
}
871+
857872
const toolCallShapeDetails = compareToolCallShape(
858873
params.openclaw.toolCalls,
859874
params.codex.toolCalls,
@@ -887,21 +902,6 @@ function classifyRuntimeParityCells(params: {
887902
};
888903
}
889904

890-
if (
891-
params.openclawScenarioStatus === "fail" ||
892-
params.codexScenarioStatus === "fail" ||
893-
!isRuntimeParityCellPassable(params.openclaw) ||
894-
!isRuntimeParityCellPassable(params.codex)
895-
) {
896-
return {
897-
drift: "failure-mode",
898-
driftDetails:
899-
params.openclawScenarioStatus === params.codexScenarioStatus
900-
? "at least one runtime failed"
901-
: `scenario status differs (${params.openclawScenarioStatus} vs ${params.codexScenarioStatus})`,
902-
};
903-
}
904-
905905
if (
906906
normalizeTextForParity(params.openclaw.finalText) ===
907907
normalizeTextForParity(params.codex.finalText)

‎extensions/qa-lab/src/runtime-tool-fixture.test.ts‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,69 @@ async function writeLiveRuntimeToolEvidence(env: QaSuiteRuntimeEnv, toolName = "
9898
]);
9999
}
100100

101+
async function runMockRuntimeToolFixtureWithOutputs(params: {
102+
toolName: string;
103+
happyArgs: Record<string, unknown>;
104+
failureArgs: Record<string, unknown>;
105+
happyOutput: string;
106+
failureOutput: string;
107+
}) {
108+
const env = await makeEnv({
109+
mock: { baseUrl: "http://127.0.0.1:9999" },
110+
});
111+
const promptSnippet = `target=${params.toolName}`;
112+
const failurePromptSnippet = `failure target=${params.toolName}`;
113+
const happyCallId = `call-${params.toolName}-happy`;
114+
const failureCallId = `call-${params.toolName}-failure`;
115+
const fetchJson = vi
116+
.fn()
117+
.mockResolvedValueOnce([])
118+
.mockResolvedValueOnce([
119+
{
120+
allInputText: promptSnippet,
121+
plannedToolCallId: happyCallId,
122+
plannedToolName: params.toolName,
123+
plannedToolArgs: params.happyArgs,
124+
},
125+
{
126+
allInputText: promptSnippet,
127+
toolOutputCallId: happyCallId,
128+
toolOutput: params.happyOutput,
129+
},
130+
{
131+
allInputText: failurePromptSnippet,
132+
plannedToolCallId: failureCallId,
133+
plannedToolName: params.toolName,
134+
plannedToolArgs: params.failureArgs,
135+
},
136+
{
137+
allInputText: failurePromptSnippet,
138+
toolOutputCallId: failureCallId,
139+
toolOutput: params.failureOutput,
140+
},
141+
]);
142+
143+
return runRuntimeToolFixture(
144+
env,
145+
{
146+
toolName: params.toolName,
147+
toolCoverage: {
148+
bucket: "openclaw-dynamic-integration",
149+
expectedLayer: "openclaw-dynamic",
150+
},
151+
promptSnippet,
152+
failurePromptSnippet,
153+
},
154+
{
155+
createSession: vi.fn(async (_env, _label, key) => key!),
156+
readEffectiveTools: vi.fn(async () => new Set([params.toolName])),
157+
runAgentPrompt: vi.fn(async () => ({})),
158+
fetchJson,
159+
ensureImageGenerationConfigured: vi.fn(),
160+
},
161+
);
162+
}
163+
101164
afterEach(async () => {
102165
await Promise.all(
103166
tempRoots.splice(0).map((tempRoot) => fs.rm(tempRoot, { recursive: true, force: true })),
@@ -595,6 +658,59 @@ describe("runtime tool fixture", () => {
595658
).rejects.toThrow("expected mock failure-path tool failure output for read");
596659
});
597660

661+
it.each([
662+
{
663+
name: "required-field",
664+
toolName: "sessions_spawn",
665+
happyArgs: { task: "reply ok" },
666+
happyOutput: "accepted",
667+
failureOutput: "task required",
668+
},
669+
{
670+
name: "unavailable-provider",
671+
toolName: "web_search",
672+
happyArgs: { query: "OpenClaw runtime parity fixed query" },
673+
happyOutput: "result",
674+
failureOutput: "web_search is disabled or no provider is available.",
675+
},
676+
])("accepts $name messages as mock failure fixture output", async (fixture) => {
677+
const details = await runMockRuntimeToolFixtureWithOutputs({
678+
...fixture,
679+
failureArgs: { __qaFailureMode: "denied-input" },
680+
});
681+
682+
expect(details).toContain(`${fixture.toolName} mock provider failure planned args`);
683+
});
684+
685+
it.each([
686+
{
687+
name: "neutral required-text",
688+
toolName: "sessions_spawn",
689+
happyArgs: { task: "reply ok" },
690+
happyOutput: "accepted",
691+
failureOutput: "no action required",
692+
expectedError: "expected mock failure-path tool failure output for sessions_spawn",
693+
},
694+
{
695+
name: "unavailable-provider happy output",
696+
toolName: "web_search",
697+
happyArgs: { query: "OpenClaw runtime parity fixed query" },
698+
happyOutput: "web_search is disabled or no provider is available.",
699+
failureOutput: "web_search is disabled or no provider is available.",
700+
expectedError: "expected mock happy-path successful tool output for web_search",
701+
},
702+
])("rejects $name as mock fixture output", async (fixture) => {
703+
await expect(
704+
runMockRuntimeToolFixtureWithOutputs({
705+
toolName: fixture.toolName,
706+
happyArgs: fixture.happyArgs,
707+
failureArgs: { __qaFailureMode: "denied-input" },
708+
happyOutput: fixture.happyOutput,
709+
failureOutput: fixture.failureOutput,
710+
}),
711+
).rejects.toThrow(fixture.expectedError);
712+
});
713+
598714
it("allows successful happy-path tool output to mention errors", async () => {
599715
const env = await makeEnv({
600716
mock: { baseUrl: "http://127.0.0.1:9999" },

‎extensions/qa-lab/src/runtime-tool-fixture.ts‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,7 @@ function isHardFailureToolOutputText(text: string) {
107107
return (
108108
/\b(?:ENOENT|EACCES|EPERM)\b/u.test(text) ||
109109
/(?:^|\n)\s*(?:Error|Exception|Failed):/u.test(text) ||
110-
/\b(?:no such file|permission denied|forbidden)\b/iu.test(text)
110+
/\b(?:disabled|forbidden|no provider|no such file|permission denied|unavailable)\b/iu.test(text)
111111
);
112112
}
113113

@@ -239,6 +239,12 @@ function readBooleanTrue(value: unknown) {
239239
return value === true;
240240
}
241241

242+
const FAILURE_LIKE_TOOL_RESULT_RE =
243+
/\b(?:denied|enoent|error|exception|fail(?:ed|ure)?|forbidden|invalid|missing|not found|permission)\b/iu;
244+
245+
const REQUIRED_FIELD_TOOL_RESULT_RE =
246+
/(?:^|[\n:,({[]\s*)["']?[A-Z_][A-Z0-9_.[\]-]*["']?\s+(?:is\s+)?required\b/iu;
247+
242248
function isFailureLikeToolResult(params: {
243249
type?: string;
244250
text: string;
@@ -247,9 +253,9 @@ function isFailureLikeToolResult(params: {
247253
}) {
248254
return (
249255
isStructuredFailureToolResult(params) ||
250-
/\b(?:denied|enoent|error|exception|fail(?:ed|ure)?|forbidden|invalid|missing|not found|permission)\b/iu.test(
251-
params.text,
252-
)
256+
isHardFailureToolOutputText(params.text) ||
257+
FAILURE_LIKE_TOOL_RESULT_RE.test(params.text) ||
258+
REQUIRED_FIELD_TOOL_RESULT_RE.test(params.text)
253259
);
254260
}
255261

0 commit comments

Comments
 (0)