Skip to content

Commit 51e0997

Browse files
vincentkocvincentkoc
committed
test(qa): accept async image fixture coverage
(cherry picked from commit 9b703d0)
1 parent 31f58cd commit 51e0997

5 files changed

Lines changed: 335 additions & 42 deletions

File tree

‎extensions/qa-lab/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
"@openclaw/plugin-sdk": "workspace:*",
1717
"@openclaw/slack": "workspace:*",
1818
"@openclaw/whatsapp": "workspace:*",
19-
"@openclaw/crabline": "0.1.5",
19+
"@openclaw/crabline": "0.1.6",
2020
"openclaw": "2026.5.28"
2121
},
2222
"peerDependencies": {

‎extensions/qa-lab/src/runtime-tool-fixture.test.ts‎

Lines changed: 247 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -776,6 +776,253 @@ describe("runtime tool fixture", () => {
776776
expect(details).toContain("read mock provider failure planned args");
777777
});
778778

779+
it("accepts non-required mock fixtures when both paths are planned without direct output", async () => {
780+
const env = await makeEnv({
781+
mock: { baseUrl: "http://127.0.0.1:9999" },
782+
});
783+
const fetchJson = vi
784+
.fn()
785+
.mockResolvedValueOnce([])
786+
.mockResolvedValueOnce([
787+
{
788+
allInputText: "target=image_generate",
789+
plannedToolCallId: "call-image-happy",
790+
plannedToolName: "image_generate",
791+
plannedToolArgs: { prompt: "QA lighthouse", filename: "runtime-tool-fixture" },
792+
},
793+
{
794+
allInputText: "failure target=image_generate",
795+
plannedToolCallId: "call-image-failure",
796+
plannedToolName: "image_generate",
797+
plannedToolArgs: { __qaFailureMode: "denied-input" },
798+
},
799+
]);
800+
801+
const details = await runRuntimeToolFixture(
802+
env,
803+
{
804+
toolName: "image_generate",
805+
toolCoverage: {
806+
bucket: "openclaw-dynamic-integration",
807+
expectedLayer: "openclaw-dynamic",
808+
required: false,
809+
action: "optional runtime parity gate with async image completion coverage",
810+
},
811+
promptSnippet: "target=image_generate",
812+
failurePromptSnippet: "failure target=image_generate",
813+
},
814+
{
815+
createSession: vi.fn(async (_env, _label, key) => key!),
816+
readEffectiveTools: vi.fn(async () => new Set(["image_generate"])),
817+
runAgentPrompt: vi.fn(async () => ({})),
818+
fetchJson,
819+
ensureImageGenerationConfigured: vi.fn(),
820+
},
821+
);
822+
823+
expect(details).toContain("image_generate mock provider report-only");
824+
expect(details).toContain("image_generate mock provider happy planned args");
825+
expect(details).toContain("image_generate mock provider failure planned args");
826+
});
827+
828+
it("still rejects failed happy output for non-required mock fixtures", async () => {
829+
const env = await makeEnv({
830+
mock: { baseUrl: "http://127.0.0.1:9999" },
831+
});
832+
const fetchJson = vi
833+
.fn()
834+
.mockResolvedValueOnce([])
835+
.mockResolvedValueOnce([
836+
{
837+
allInputText: "target=image_generate",
838+
plannedToolCallId: "call-image-happy",
839+
plannedToolName: "image_generate",
840+
plannedToolArgs: { prompt: "QA lighthouse" },
841+
},
842+
{
843+
allInputText: "target=image_generate",
844+
toolOutputCallId: "call-image-happy",
845+
toolOutput: "Failed: provider rejected image request",
846+
},
847+
{
848+
allInputText: "failure target=image_generate",
849+
plannedToolCallId: "call-image-failure",
850+
plannedToolName: "image_generate",
851+
plannedToolArgs: { __qaFailureMode: "denied-input" },
852+
},
853+
]);
854+
855+
await expect(
856+
runRuntimeToolFixture(
857+
env,
858+
{
859+
toolName: "image_generate",
860+
toolCoverage: {
861+
bucket: "openclaw-dynamic-integration",
862+
expectedLayer: "openclaw-dynamic",
863+
required: false,
864+
action: "optional runtime parity gate with async image completion coverage",
865+
},
866+
promptSnippet: "target=image_generate",
867+
failurePromptSnippet: "failure target=image_generate",
868+
},
869+
{
870+
createSession: vi.fn(async (_env, _label, key) => key!),
871+
readEffectiveTools: vi.fn(async () => new Set(["image_generate"])),
872+
runAgentPrompt: vi.fn(async () => ({})),
873+
fetchJson,
874+
ensureImageGenerationConfigured: vi.fn(),
875+
},
876+
),
877+
).rejects.toThrow("expected mock happy-path successful tool output for image_generate");
878+
});
879+
880+
it("still rejects successful failure output for non-required mock fixtures", async () => {
881+
const env = await makeEnv({
882+
mock: { baseUrl: "http://127.0.0.1:9999" },
883+
});
884+
const fetchJson = vi
885+
.fn()
886+
.mockResolvedValueOnce([])
887+
.mockResolvedValueOnce([
888+
{
889+
allInputText: "target=image_generate",
890+
plannedToolCallId: "call-image-happy",
891+
plannedToolName: "image_generate",
892+
plannedToolArgs: { prompt: "QA lighthouse" },
893+
},
894+
{
895+
allInputText: "failure target=image_generate",
896+
plannedToolCallId: "call-image-failure",
897+
plannedToolName: "image_generate",
898+
plannedToolArgs: { __qaFailureMode: "denied-input" },
899+
},
900+
{
901+
allInputText: "failure target=image_generate",
902+
toolOutputCallId: "call-image-failure",
903+
toolOutput: "Task queued for async image delivery",
904+
},
905+
]);
906+
907+
await expect(
908+
runRuntimeToolFixture(
909+
env,
910+
{
911+
toolName: "image_generate",
912+
toolCoverage: {
913+
bucket: "openclaw-dynamic-integration",
914+
expectedLayer: "openclaw-dynamic",
915+
required: false,
916+
action: "optional runtime parity gate with async image completion coverage",
917+
},
918+
promptSnippet: "target=image_generate",
919+
failurePromptSnippet: "failure target=image_generate",
920+
},
921+
{
922+
createSession: vi.fn(async (_env, _label, key) => key!),
923+
readEffectiveTools: vi.fn(async () => new Set(["image_generate"])),
924+
runAgentPrompt: vi.fn(async () => ({})),
925+
fetchJson,
926+
ensureImageGenerationConfigured: vi.fn(),
927+
},
928+
),
929+
).rejects.toThrow("expected mock failure-path tool failure output for image_generate");
930+
});
931+
932+
it("rejects malformed report-only failure plans for non-required mock fixtures", async () => {
933+
const env = await makeEnv({
934+
mock: { baseUrl: "http://127.0.0.1:9999" },
935+
});
936+
const fetchJson = vi
937+
.fn()
938+
.mockResolvedValueOnce([])
939+
.mockResolvedValueOnce([
940+
{
941+
allInputText: "target=image_generate",
942+
plannedToolCallId: "call-image-happy",
943+
plannedToolName: "image_generate",
944+
plannedToolArgs: { prompt: "QA lighthouse" },
945+
},
946+
{
947+
allInputText: "failure target=image_generate",
948+
plannedToolCallId: "call-image-failure",
949+
plannedToolName: "image_generate",
950+
plannedToolArgs: { prompt: "not a denied-input failure" },
951+
},
952+
]);
953+
954+
await expect(
955+
runRuntimeToolFixture(
956+
env,
957+
{
958+
toolName: "image_generate",
959+
toolCoverage: {
960+
bucket: "openclaw-dynamic-integration",
961+
expectedLayer: "openclaw-dynamic",
962+
required: false,
963+
action: "optional runtime parity gate with async image completion coverage",
964+
},
965+
promptSnippet: "target=image_generate",
966+
failurePromptSnippet: "failure target=image_generate",
967+
},
968+
{
969+
createSession: vi.fn(async (_env, _label, key) => key!),
970+
readEffectiveTools: vi.fn(async () => new Set(["image_generate"])),
971+
runAgentPrompt: vi.fn(async () => ({})),
972+
fetchJson,
973+
ensureImageGenerationConfigured: vi.fn(),
974+
},
975+
),
976+
).rejects.toThrow("expected mock failure-path denied-input args for image_generate");
977+
});
978+
979+
it("rejects malformed report-only happy plans for non-required mock fixtures", async () => {
980+
const env = await makeEnv({
981+
mock: { baseUrl: "http://127.0.0.1:9999" },
982+
});
983+
const fetchJson = vi
984+
.fn()
985+
.mockResolvedValueOnce([])
986+
.mockResolvedValueOnce([
987+
{
988+
allInputText: "target=image_generate",
989+
plannedToolCallId: "call-image-happy",
990+
plannedToolName: "image_generate",
991+
plannedToolArgs: {},
992+
},
993+
{
994+
allInputText: "failure target=image_generate",
995+
plannedToolCallId: "call-image-failure",
996+
plannedToolName: "image_generate",
997+
plannedToolArgs: { __qaFailureMode: "denied-input" },
998+
},
999+
]);
1000+
1001+
await expect(
1002+
runRuntimeToolFixture(
1003+
env,
1004+
{
1005+
toolName: "image_generate",
1006+
toolCoverage: {
1007+
bucket: "openclaw-dynamic-integration",
1008+
expectedLayer: "openclaw-dynamic",
1009+
required: false,
1010+
action: "optional runtime parity gate with async image completion coverage",
1011+
},
1012+
promptSnippet: "target=image_generate",
1013+
failurePromptSnippet: "failure target=image_generate",
1014+
},
1015+
{
1016+
createSession: vi.fn(async (_env, _label, key) => key!),
1017+
readEffectiveTools: vi.fn(async () => new Set(["image_generate"])),
1018+
runAgentPrompt: vi.fn(async () => ({})),
1019+
fetchJson,
1020+
ensureImageGenerationConfigured: vi.fn(),
1021+
},
1022+
),
1023+
).rejects.toThrow("expected mock happy-path prompt args for image_generate");
1024+
});
1025+
7791026
it("rejects failure-shaped mock happy-path tool output", async () => {
7801027
const env = await makeEnv({
7811028
mock: { baseUrl: "http://127.0.0.1:9999" },

‎extensions/qa-lab/src/runtime-tool-fixture.ts‎

Lines changed: 68 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,10 @@
22
import fs from "node:fs/promises";
33
import path from "node:path";
44
import { isRecord } from "openclaw/plugin-sdk/string-coerce-runtime";
5-
import { readRuntimeToolCoverageMetadata } from "./runtime-tool-metadata.js";
5+
import {
6+
type QaRuntimeToolCoverageMetadata,
7+
readRuntimeToolCoverageMetadata,
8+
} from "./runtime-tool-metadata.js";
69
import { liveTurnTimeoutMs } from "./suite-runtime-agent-common.js";
710
import { readRawQaSessionStore } from "./suite-runtime-agent-session.js";
811
import type { QaSuiteRuntimeEnv } from "./suite-runtime-types.js";
@@ -555,6 +558,37 @@ function formatCodexNativeWorkspaceDetails(params: {
555558
.join("\n");
556559
}
557560

561+
function formatReportOnlyMockDetails(params: {
562+
toolName: string;
563+
happyRequest: QaRuntimeToolFixtureRequest;
564+
failureRequest: QaRuntimeToolFixtureRequest;
565+
}) {
566+
return [
567+
`${params.toolName} mock provider report-only: direct tool output is not required by this fixture`,
568+
`${params.toolName} mock provider happy planned args (diagnostic only): ${formatPlannedToolArgs(params.happyRequest.plannedToolArgs)}`,
569+
`${params.toolName} mock provider failure planned args (diagnostic only): ${formatPlannedToolArgs(params.failureRequest.plannedToolArgs)}`,
570+
].join("\n");
571+
}
572+
573+
function isAsyncReportOnlyMockCoverage(metadata: QaRuntimeToolCoverageMetadata) {
574+
return !metadata.required && /\basync\b/iu.test(metadata.action ?? "");
575+
}
576+
577+
function plannedRequestHasDeniedInputFailure(request: QaRuntimeToolFixtureRequest) {
578+
return (
579+
isRecord(request.plannedToolArgs) &&
580+
request.plannedToolArgs["__qaFailureMode"] === "denied-input"
581+
);
582+
}
583+
584+
function plannedRequestHasPrompt(request: QaRuntimeToolFixtureRequest) {
585+
return (
586+
isRecord(request.plannedToolArgs) &&
587+
typeof request.plannedToolArgs.prompt === "string" &&
588+
request.plannedToolArgs.prompt.trim().length > 0
589+
);
590+
}
591+
558592
function formatKnownHarnessGapDetails(toolName: string, config: QaRuntimeToolFixtureConfig) {
559593
const knownHarnessGap = isKnownHarnessGap(config.knownHarnessGap) ? config.knownHarnessGap : {};
560594
const issue = readString(knownHarnessGap.issue);
@@ -722,6 +756,39 @@ export async function runRuntimeToolFixture(
722756
excludedPromptSnippet: failurePromptSnippet,
723757
toolName,
724758
});
759+
const failurePlannedRequest = findPlannedRequest({
760+
requests,
761+
requestCountBefore,
762+
promptSnippet: failurePromptSnippet,
763+
toolName,
764+
});
765+
const failureRequest = findExecutedRequest({
766+
requests,
767+
requestCountBefore,
768+
promptSnippet: failurePromptSnippet,
769+
toolName,
770+
});
771+
if (
772+
isAsyncReportOnlyMockCoverage(metadata) &&
773+
happyPlannedRequest &&
774+
failurePlannedRequest &&
775+
!happyRequest
776+
) {
777+
if (!plannedRequestHasPrompt(happyPlannedRequest)) {
778+
throw new Error(`expected mock happy-path prompt args for ${toolName}`);
779+
}
780+
if (!plannedRequestHasDeniedInputFailure(failurePlannedRequest)) {
781+
throw new Error(`expected mock failure-path denied-input args for ${toolName}`);
782+
}
783+
if (failureRequest && !requestHasFailureLikeToolOutput(failureRequest.outputRequest)) {
784+
throw new Error(`expected mock failure-path tool failure output for ${toolName}`);
785+
}
786+
return formatReportOnlyMockDetails({
787+
toolName,
788+
happyRequest: happyPlannedRequest,
789+
failureRequest: failurePlannedRequest,
790+
});
791+
}
725792
// Async runtime tools prove the start call here; completion is covered by
726793
// their task lifecycle scenarios.
727794
const happyPlannedOnly = Boolean(happyPlannedRequest && !happyPathOutputRequired);
@@ -749,18 +816,6 @@ export async function runRuntimeToolFixture(
749816
}
750817
throw new Error(`expected mock happy-path successful tool output for ${toolName}`);
751818
}
752-
const failurePlannedRequest = findPlannedRequest({
753-
requests,
754-
requestCountBefore,
755-
promptSnippet: failurePromptSnippet,
756-
toolName,
757-
});
758-
const failureRequest = findExecutedRequest({
759-
requests,
760-
requestCountBefore,
761-
promptSnippet: failurePromptSnippet,
762-
toolName,
763-
});
764819
if (!failureRequest) {
765820
if (dynamicExposureIntentionallyExcluded) {
766821
return formatCodexNativeWorkspaceDetails({

0 commit comments

Comments
 (0)