Skip to content

Commit 8242923

Browse files
vincentkocvincentkoc
committed
fix(qa): allow async runtime fixture starts
1 parent 414c250 commit 8242923

3 files changed

Lines changed: 264 additions & 14 deletions

File tree

‎extensions/qa-lab/src/runtime-tool-fixture.test.ts‎

Lines changed: 228 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -268,6 +268,115 @@ describe("runtime tool fixture", () => {
268268
expect(details).toContain("read live provider failure planned args");
269269
});
270270

271+
it("allows async live runtime tool fixtures to prove the happy path with the planned call", async () => {
272+
const env = await makeEnv();
273+
await writeQaSessionTranscript(env, "agent:qa:runtime-tool:image_generate:happy", [
274+
{
275+
role: "assistant",
276+
content: [
277+
{
278+
type: "tool_use",
279+
id: "call-image-happy",
280+
name: "image_generate",
281+
input: { prompt: "QA lighthouse runtime parity fixture" },
282+
},
283+
],
284+
},
285+
]);
286+
await writeQaSessionTranscript(env, "agent:qa:runtime-tool:image_generate:failure", [
287+
{
288+
role: "assistant",
289+
content: [
290+
{
291+
type: "tool_use",
292+
id: "call-image-failure",
293+
name: "image_generate",
294+
input: { __qaFailureMode: "denied-input" },
295+
},
296+
],
297+
},
298+
{
299+
role: "tool",
300+
toolName: "image_generate",
301+
tool_call_id: "call-image-failure",
302+
isError: true,
303+
content: "denied-input",
304+
},
305+
]);
306+
307+
const details = await runRuntimeToolFixture(
308+
env,
309+
{
310+
toolName: "image_generate",
311+
toolCoverage: {
312+
bucket: "openclaw-dynamic-integration",
313+
expectedLayer: "openclaw-dynamic",
314+
},
315+
happyPathOutputRequired: false,
316+
},
317+
{
318+
createSession: vi.fn(async (_env, _label, key) => key!),
319+
readEffectiveTools: vi.fn(async () => new Set(["image_generate"])),
320+
runAgentPrompt: vi.fn(async () => ({})),
321+
fetchJson: vi.fn(),
322+
ensureImageGenerationConfigured: vi.fn(),
323+
},
324+
);
325+
326+
expect(details).toContain(
327+
"image_generate live provider happy direct output not required for this async fixture",
328+
);
329+
expect(details).toContain("image_generate live provider failure planned args");
330+
});
331+
332+
it("still requires async live runtime tool fixtures to call the happy-path tool", async () => {
333+
const env = await makeEnv();
334+
await writeQaSessionTranscript(env, "agent:qa:runtime-tool:image_generate:happy", [
335+
{ role: "assistant", content: "I can start image generation later." },
336+
]);
337+
await writeQaSessionTranscript(env, "agent:qa:runtime-tool:image_generate:failure", [
338+
{
339+
role: "assistant",
340+
content: [
341+
{
342+
type: "tool_use",
343+
id: "call-image-failure",
344+
name: "image_generate",
345+
input: { __qaFailureMode: "denied-input" },
346+
},
347+
],
348+
},
349+
{
350+
role: "tool",
351+
toolName: "image_generate",
352+
tool_call_id: "call-image-failure",
353+
isError: true,
354+
content: "denied-input",
355+
},
356+
]);
357+
358+
await expect(
359+
runRuntimeToolFixture(
360+
env,
361+
{
362+
toolName: "image_generate",
363+
toolCoverage: {
364+
bucket: "openclaw-dynamic-integration",
365+
expectedLayer: "openclaw-dynamic",
366+
},
367+
happyPathOutputRequired: false,
368+
},
369+
{
370+
createSession: vi.fn(async (_env, _label, key) => key!),
371+
readEffectiveTools: vi.fn(async () => new Set(["image_generate"])),
372+
runAgentPrompt: vi.fn(async () => ({})),
373+
fetchJson: vi.fn(),
374+
ensureImageGenerationConfigured: vi.fn(),
375+
},
376+
),
377+
).rejects.toThrow("expected live happy-path tool call for image_generate");
378+
});
379+
271380
it("requires live failure fixtures to produce failure-shaped tool output", async () => {
272381
const env = await makeEnv();
273382
await writeQaSessionTranscript(env, "agent:qa:runtime-tool:read:happy", [
@@ -445,6 +554,70 @@ describe("runtime tool fixture", () => {
445554
expect(details).toContain("mock provider happy planned args (diagnostic only)");
446555
});
447556

557+
it("reports Codex-native async planned-only happy fixtures without dereferencing missing output", async () => {
558+
const env = await makeEnv({
559+
mock: { baseUrl: "http://127.0.0.1:9999" },
560+
gateway: {
561+
baseUrl: "http://127.0.0.1:1",
562+
tempRoot: "",
563+
workspaceDir: "",
564+
runtimeEnv: { OPENCLAW_QA_FORCE_RUNTIME: "codex" },
565+
call: vi.fn(),
566+
},
567+
});
568+
env.gateway.tempRoot = env.repoRoot;
569+
env.gateway.workspaceDir = env.repoRoot;
570+
571+
const fetchJson = vi
572+
.fn()
573+
.mockResolvedValueOnce([])
574+
.mockResolvedValueOnce([
575+
{
576+
allInputText: "target=image_generate",
577+
plannedToolCallId: "call-image-happy",
578+
plannedToolName: "image_generate",
579+
plannedToolArgs: { prompt: "QA lighthouse runtime parity fixture" },
580+
},
581+
{
582+
allInputText: "failure target=image_generate",
583+
plannedToolCallId: "call-image-failure",
584+
plannedToolName: "image_generate",
585+
plannedToolArgs: { __qaFailureMode: "denied-input" },
586+
},
587+
{
588+
allInputText: "failure target=image_generate",
589+
toolOutputCallId: "call-image-failure",
590+
toolOutput: "Error: denied-input",
591+
},
592+
]);
593+
594+
const details = await runRuntimeToolFixture(
595+
env,
596+
{
597+
toolName: "image_generate",
598+
toolCoverage: {
599+
bucket: "codex-native-workspace",
600+
expectedLayer: "codex-native-workspace",
601+
reason: "Codex owns image generation natively in this fixture.",
602+
},
603+
promptSnippet: "target=image_generate",
604+
failurePromptSnippet: "failure target=image_generate",
605+
happyPathOutputRequired: false,
606+
},
607+
{
608+
createSession: vi.fn(async (_env, _label, key) => key!),
609+
readEffectiveTools: vi.fn(async () => new Set<string>()),
610+
runAgentPrompt: vi.fn(async () => ({})),
611+
fetchJson,
612+
ensureImageGenerationConfigured: vi.fn(),
613+
},
614+
);
615+
616+
expect(details).toContain("codex-native-workspace image_generate");
617+
expect(details).toContain('"prompt":"QA lighthouse runtime parity fixture"');
618+
expect(details).toContain('"__qaFailureMode":"denied-input"');
619+
});
620+
448621
it("requires mock runtime tool fixtures to produce tool output", async () => {
449622
const env = await makeEnv({
450623
mock: { baseUrl: "http://127.0.0.1:9999" },
@@ -492,6 +665,61 @@ describe("runtime tool fixture", () => {
492665
).rejects.toThrow("expected mock happy-path tool output for read");
493666
});
494667

668+
it("allows async mock runtime tool fixtures to prove the happy path with the planned call", async () => {
669+
const env = await makeEnv({
670+
mock: { baseUrl: "http://127.0.0.1:9999" },
671+
});
672+
const fetchJson = vi
673+
.fn()
674+
.mockResolvedValueOnce([])
675+
.mockResolvedValueOnce([
676+
{
677+
allInputText: "target=image_generate",
678+
plannedToolCallId: "call-image-happy",
679+
plannedToolName: "image_generate",
680+
plannedToolArgs: { prompt: "QA lighthouse runtime parity fixture" },
681+
},
682+
{
683+
allInputText: "failure target=image_generate",
684+
plannedToolCallId: "call-image-failure",
685+
plannedToolName: "image_generate",
686+
plannedToolArgs: { __qaFailureMode: "denied-input" },
687+
},
688+
{
689+
allInputText: "failure target=image_generate",
690+
toolOutputCallId: "call-image-failure",
691+
toolOutput: "Error: denied-input",
692+
},
693+
]);
694+
695+
const details = await runRuntimeToolFixture(
696+
env,
697+
{
698+
toolName: "image_generate",
699+
toolCoverage: {
700+
bucket: "openclaw-dynamic-integration",
701+
expectedLayer: "openclaw-dynamic",
702+
},
703+
promptSnippet: "target=image_generate",
704+
failurePromptSnippet: "failure target=image_generate",
705+
happyPathOutputRequired: false,
706+
},
707+
{
708+
createSession: vi.fn(async (_env, _label, key) => key!),
709+
readEffectiveTools: vi.fn(async () => new Set(["image_generate"])),
710+
runAgentPrompt: vi.fn(async () => ({})),
711+
fetchJson,
712+
ensureImageGenerationConfigured: vi.fn(),
713+
},
714+
);
715+
716+
expect(details).toContain(
717+
"image_generate mock provider happy direct output not required for this async fixture",
718+
);
719+
expect(details).toContain('"prompt":"QA lighthouse runtime parity fixture"');
720+
expect(details).toContain('"__qaFailureMode":"denied-input"');
721+
});
722+
495723
it("accepts mock runtime tool fixtures only after planned calls return output", async () => {
496724
const env = await makeEnv({
497725
mock: { baseUrl: "http://127.0.0.1:9999" },

‎extensions/qa-lab/src/runtime-tool-fixture.ts‎

Lines changed: 35 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ type QaRuntimeToolFixtureConfig = Record<string, unknown> & {
1313
failurePrompt?: unknown;
1414
promptSnippet?: unknown;
1515
failurePromptSnippet?: unknown;
16+
happyPathOutputRequired?: unknown;
1617
ensureImageGeneration?: unknown;
1718
expectedAvailable?: unknown;
1819
toolCoverage?: unknown;
@@ -627,6 +628,7 @@ export async function runRuntimeToolFixture(
627628
config.failurePromptSnippet,
628629
`failure target=${toolName}`,
629630
);
631+
const happyPathOutputRequired = readBoolean(config.happyPathOutputRequired, true);
630632
const requestCountBefore = env.mock
631633
? readQaRuntimeToolFixtureRequests(await deps.fetchJson(`${env.mock.baseUrl}/debug/requests`))
632634
.length
@@ -650,16 +652,22 @@ export async function runRuntimeToolFixture(
650652
toolName,
651653
});
652654
if (!happyRequest.outputRequest) {
653-
if (isKnownHarnessGap(config.knownHarnessGap)) {
654-
return formatKnownHarnessGapDetails(toolName, config);
655+
const happyPlannedOnly = happyRequest.plannedRequest && !happyPathOutputRequired;
656+
if (happyPlannedOnly) {
657+
// Async runtime tools prove the start call here; completion is covered
658+
// by their task lifecycle scenarios.
659+
} else {
660+
if (isKnownHarnessGap(config.knownHarnessGap)) {
661+
return formatKnownHarnessGapDetails(toolName, config);
662+
}
663+
throw new Error(
664+
happyRequest.plannedRequest
665+
? `expected live happy-path tool output for ${toolName}`
666+
: `expected live happy-path tool call for ${toolName}`,
667+
);
655668
}
656-
throw new Error(
657-
happyRequest.plannedRequest
658-
? `expected live happy-path tool output for ${toolName}`
659-
: `expected live happy-path tool call for ${toolName}`,
660-
);
661669
}
662-
if (happyRequest.outputRequest.structuredFailure) {
670+
if (happyRequest.outputRequest?.structuredFailure) {
663671
if (isKnownHarnessGap(config.knownHarnessGap)) {
664672
return formatKnownHarnessGapDetails(toolName, config);
665673
}
@@ -688,8 +696,13 @@ export async function runRuntimeToolFixture(
688696
}
689697
return [
690698
`${toolName} live provider happy planned args (diagnostic only): ${JSON.stringify(happyRequest.plannedRequest?.args ?? {})}`,
699+
happyPathOutputRequired
700+
? undefined
701+
: `${toolName} live provider happy direct output not required for this async fixture`,
691702
`${toolName} live provider failure planned args (diagnostic only): ${JSON.stringify(failureRequest.plannedRequest?.args ?? {})}`,
692-
].join("\n");
703+
]
704+
.filter(Boolean)
705+
.join("\n");
693706
}
694707

695708
const requests = readQaRuntimeToolFixtureRequests(
@@ -709,7 +722,10 @@ export async function runRuntimeToolFixture(
709722
excludedPromptSnippet: failurePromptSnippet,
710723
toolName,
711724
});
712-
if (!happyRequest) {
725+
// Async runtime tools prove the start call here; completion is covered by
726+
// their task lifecycle scenarios.
727+
const happyPlannedOnly = Boolean(happyPlannedRequest && !happyPathOutputRequired);
728+
if (!happyRequest && !happyPlannedOnly) {
713729
if (dynamicExposureIntentionallyExcluded) {
714730
return formatCodexNativeWorkspaceDetails({
715731
toolName,
@@ -727,7 +743,7 @@ export async function runRuntimeToolFixture(
727743
: `expected mock happy-path request for ${toolName}`,
728744
);
729745
}
730-
if (requestHasHappyPathFailureToolOutput(happyRequest.outputRequest)) {
746+
if (happyRequest && requestHasHappyPathFailureToolOutput(happyRequest.outputRequest)) {
731747
if (isKnownHarnessGap(config.knownHarnessGap)) {
732748
return formatKnownHarnessGapDetails(toolName, config);
733749
}
@@ -776,13 +792,18 @@ export async function runRuntimeToolFixture(
776792
toolName,
777793
tools,
778794
reason: metadata.reason,
779-
happyRequest: happyRequest.plannedRequest,
795+
happyRequest: happyRequest?.plannedRequest ?? happyPlannedRequest,
780796
failureRequest: failureRequest.plannedRequest,
781797
});
782798
}
783799

784800
return [
785-
`${toolName} mock provider happy planned args (diagnostic only): ${formatPlannedToolArgs(happyRequest.plannedRequest.plannedToolArgs)}`,
801+
`${toolName} mock provider happy planned args (diagnostic only): ${formatPlannedToolArgs((happyRequest?.plannedRequest ?? happyPlannedRequest)?.plannedToolArgs)}`,
802+
happyPathOutputRequired
803+
? undefined
804+
: `${toolName} mock provider happy direct output not required for this async fixture`,
786805
`${toolName} mock provider failure planned args (diagnostic only): ${formatPlannedToolArgs(failureRequest.plannedRequest.plannedToolArgs)}`,
787-
].join("\n");
806+
]
807+
.filter(Boolean)
808+
.join("\n");
788809
}

‎qa/scenarios/runtime/tools/image-generate.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ scenario:
3939
reason: image_generate is an OpenClaw integration tool whose happy path yields for async completion, so standard direct call/result parity would compare different lifecycle phases.
4040
promptSnippet: "target=image_generate"
4141
failurePromptSnippet: "failure target=image_generate"
42+
happyPathOutputRequired: false
4243

4344
flow:
4445
steps:

0 commit comments

Comments
 (0)