Skip to content

Commit 2660d46

Browse files
RomneyDaRomneyDa
committed
test: add hosted media provider QA live evidence
1 parent 80a3c51 commit 2660d46

8 files changed

Lines changed: 740 additions & 269 deletions

‎package.json‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1880,10 +1880,10 @@
18801880
"test:live:codex-harness": "node scripts/test-live.mjs --codex-harness -- src/gateway/gateway-codex-harness.live.test.ts",
18811881
"test:live:crestodian-rescue-channel": "node scripts/run-with-env.mjs OPENCLAW_LIVE_CRESTODIAN_RESCUE_CHANNEL=1 -- node scripts/test-live.mjs -- src/crestodian/rescue-channel.live.test.ts",
18821882
"test:live:gateway-profiles": "node scripts/test-live.mjs -- src/gateway/gateway-models.profiles.live.test.ts",
1883-
"test:live:media": "node --import tsx scripts/test-live-media.ts",
1884-
"test:live:media:image": "node --import tsx scripts/test-live-media.ts image",
1885-
"test:live:media:music": "node --import tsx scripts/test-live-media.ts music",
1886-
"test:live:media:video": "node --import tsx scripts/test-live-media.ts video",
1883+
"test:live:media": "node --import tsx test/e2e/qa-lab/media/hosted-media-provider-live.ts",
1884+
"test:live:media:image": "node --import tsx test/e2e/qa-lab/media/hosted-media-provider-live.ts image",
1885+
"test:live:media:music": "node --import tsx test/e2e/qa-lab/media/hosted-media-provider-live.ts music",
1886+
"test:live:media:video": "node --import tsx test/e2e/qa-lab/media/hosted-media-provider-live.ts video",
18871887
"test:live:models-profiles": "node scripts/test-live.mjs -- src/agents/models.profiles.live.test.ts",
18881888
"test:macos:ci": "node scripts/test-projects.mjs src/daemon/launchd.test.ts src/daemon/runtime-paths.test.ts src/daemon/runtime-binary.test.ts src/infra/brew.test.ts src/infra/stable-node-path.test.ts test/scripts/vitest-process-group.test.ts test/scripts/package-mac-app.test.ts test/scripts/package-mac-dist.test.ts test/scripts/create-dmg.test.ts test/scripts/codesign-mac-app.test.ts test/scripts/notarize-mac-artifact.test.ts",
18891889
"test:max": "node scripts/test-projects-max.mjs",

‎qa/scenarios/media/hosted-image-generation-providers-live.yaml‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
title: Hosted image generation providers live
2+
3+
scenario:
4+
id: hosted-image-generation-providers-live
5+
surface: long-tail-hosted-providers
6+
category: long-tail-hosted-providers.hosted-media-providers
7+
runtimeParityTier: live-only
8+
coverage:
9+
primary:
10+
- hosted-providers.image-generation-providers
11+
objective: Verify hosted image generation providers can generate real images through the live provider runtime sweep.
12+
successCriteria:
13+
- The QA producer runs the canonical live media image harness instead of duplicating provider assertions.
14+
- At least one auth-available hosted image provider runs through `provider.generateImage`.
15+
- Generated image output includes image MIME type and non-trivial bytes.
16+
- Missing live provider credentials produce blocked evidence rather than false passing primary coverage.
17+
docsRefs:
18+
- docs/help/testing.md
19+
- docs/tools/image-generation.md
20+
- docs/tools/media-overview.md
21+
codeRefs:
22+
- test/e2e/qa-lab/media/hosted-media-provider-live.ts
23+
- test/image-generation.runtime.live.test.ts
24+
- src/image-generation/live-test-helpers.ts
25+
execution:
26+
kind: script
27+
path: test/e2e/qa-lab/media/hosted-media-provider-live.ts
28+
summary: Runs the existing hosted image-generation live provider sweep and emits QA Lab evidence.
29+
allowBlockedEvidence: true
30+
timeoutMs: 7200000
31+
args:
32+
- --qa-evidence
33+
- --suite
34+
- image
35+
- --artifact-base
36+
- ${outputDir}

‎qa/scenarios/media/hosted-video-generation-providers-live.yaml‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
title: Hosted video generation providers live
2+
3+
scenario:
4+
id: hosted-video-generation-providers-live
5+
surface: long-tail-hosted-providers
6+
category: long-tail-hosted-providers.hosted-media-providers
7+
runtimeParityTier: live-only
8+
coverage:
9+
primary:
10+
- hosted-providers.video-generation-providers
11+
- media.reference-image-video-and-audio-inputs
12+
secondary:
13+
- media.video-generation-tool-invocation
14+
objective: Verify hosted video generation providers can generate real videos and exercise full reference-input modes where supported.
15+
successCriteria:
16+
- The QA producer runs the canonical live media video harness instead of duplicating provider assertions.
17+
- At least one auth-available hosted video provider runs through `provider.generateVideo`.
18+
- Full live video modes are enabled so image-to-video and video-to-video reference inputs are attempted where provider capabilities allow them.
19+
- Generated video output includes video MIME type with bytes or an HTTP(S) URL.
20+
- Missing live provider credentials produce blocked evidence rather than false passing primary coverage.
21+
docsRefs:
22+
- docs/help/testing.md
23+
- docs/tools/video-generation.md
24+
- docs/tools/media-overview.md
25+
codeRefs:
26+
- test/e2e/qa-lab/media/hosted-media-provider-live.ts
27+
- extensions/video-generation-providers.live.test.ts
28+
- src/video-generation/runtime.ts
29+
- src/agents/tools/video-generate-tool.ts
30+
execution:
31+
kind: script
32+
path: test/e2e/qa-lab/media/hosted-media-provider-live.ts
33+
summary: Runs the existing hosted video-generation live provider sweep with full reference-input modes and emits QA Lab evidence.
34+
allowBlockedEvidence: true
35+
timeoutMs: 14400000
36+
args:
37+
- --qa-evidence
38+
- --suite
39+
- video
40+
- --artifact-base
41+
- ${outputDir}

‎scripts/test-projects.test-support.mjs‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1924,7 +1924,10 @@ const TOOLING_SOURCE_TEST_TARGETS = new Map([
19241924
["scripts/test-projects.test-support.mjs", ["test/scripts/test-projects.test.ts"]],
19251925
["scripts/dev/gateway-smoke.ts", ["test/e2e/qa-lab/runtime/gateway-smoke.e2e.test.ts"]],
19261926
["scripts/dev/test-device-pair-telegram.ts", ["test/scripts/test-device-pair-telegram.test.ts"]],
1927-
["scripts/test-live-media.ts", ["test/scripts/test-live-media.test.ts"]],
1927+
[
1928+
"test/e2e/qa-lab/media/hosted-media-provider-live.ts",
1929+
["test/e2e/qa-lab/media/hosted-media-provider-live.test.ts"],
1930+
],
19281931
["scripts/profile-extension-memory.mjs", ["test/scripts/profile-extension-memory.test.ts"]],
19291932
[
19301933
"scripts/openclaw-performance-source-summary.mjs",

‎src/scripts/test-live-media.test.ts‎

Lines changed: 0 additions & 83 deletions
This file was deleted.

0 commit comments

Comments
 (0)