Skip to content

Commit 8f7f48a

Browse files
zoomote[bot]roomote
andauthored
[Chore] Add Extension Host visual regression coverage (#1426)
* test(webview): add Extension Host visual checks * test(webview): add authoritative visual baselines * test(electron): stabilize chat visual scene * test(electron): normalize snapshot focus * test(electron): add populated chat baseline * fix(visual): stabilize review coverage * test(webview): migrate announcement link visual * test(visual): refresh pinned baselines * test(visual): cover localized settings layout * test(visual): refresh localized baselines * test(electron): refresh pinned chat baseline --------- Co-authored-by: Roomote <[email protected]>
1 parent b55ff87 commit 8f7f48a

57 files changed

Lines changed: 1131 additions & 382 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/visual-regression.yml‎

Lines changed: 44 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ on:
77
paths:
88
- "apps/vscode-e2e/package.json"
99
- "apps/vscode-e2e/src/theme-fixtures/**"
10+
- "apps/vscode-e2e/src/visual/**"
11+
- "apps/vscode-e2e/playwright-electron.config.ts"
1012
- "packages/types/src/api.ts"
1113
- "packages/types/src/vscode-extension-host.ts"
1214
- "src/core/webview/ClineProvider.ts"
@@ -32,7 +34,7 @@ jobs:
3234
runs-on: ubuntu-latest
3335
timeout-minutes: 10
3436
container:
35-
image: mcr.microsoft.com/playwright:v1.60.0-noble@sha256:9bd26ad900bb5e0f4dee75839e957a89ae89c2b7ab1e76050e559790e946b948
37+
image: mcr.microsoft.com/playwright:v1.62.1-noble@sha256:dcc5531e97840b9b5e794f2814476b21571c5124a3fca2267d73041f56e7580e
3638
options: --ipc=host
3739
steps:
3840
- name: Checkout code
@@ -91,3 +93,44 @@ jobs:
9193
run: pnpm --filter @roo-code/vscode-e2e themes:test
9294
- name: Check generated VS Code theme fixtures
9395
run: xvfb-run -a pnpm --filter @roo-code/vscode-e2e themes:check
96+
97+
extension-host-visual:
98+
runs-on: ubuntu-latest
99+
timeout-minutes: 20
100+
container:
101+
image: mcr.microsoft.com/playwright:v1.62.1-noble@sha256:dcc5531e97840b9b5e794f2814476b21571c5124a3fca2267d73041f56e7580e
102+
options: --ipc=host
103+
env:
104+
LANG: en_US.UTF-8
105+
LC_ALL: en_US.UTF-8
106+
TZ: UTC
107+
steps:
108+
- name: Checkout code
109+
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
110+
with:
111+
persist-credentials: false
112+
- name: Setup Node.js and pnpm
113+
uses: ./.github/actions/setup-node-pnpm
114+
with:
115+
install-args: "--frozen-lockfile"
116+
- name: Get pinned VS Code version
117+
id: vscode-version
118+
run: |
119+
VERSION=$(node -p 'require("./apps/vscode-e2e/package.json").devDependencies["@types/vscode"]')
120+
echo "version=$VERSION" >> "$GITHUB_OUTPUT"
121+
- name: Cache pinned VS Code
122+
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
123+
with:
124+
path: apps/vscode-e2e/.vscode-test/
125+
key: vscode-electron-visual-${{ runner.os }}-${{ steps.vscode-version.outputs.version }}-v1
126+
- name: Run Extension Host visual tests
127+
run: xvfb-run -a pnpm --filter @roo-code/vscode-e2e test:visual:electron
128+
- name: Upload Extension Host visual artifacts
129+
if: failure()
130+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
131+
with:
132+
name: extension-host-visual-regression
133+
path: |
134+
apps/vscode-e2e/playwright-report-electron
135+
apps/vscode-e2e/test-results-electron
136+
if-no-files-found: ignore

‎.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@ AGENTS.local.md
2323
# Test environment
2424
.test_env
2525
.vscode-test/
26+
apps/vscode-e2e/playwright-report-electron/
27+
apps/vscode-e2e/test-results-electron/
2628

2729
# Docs
2830
docs/_site/

‎apps/vscode-e2e/AGENTS.md‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,19 @@ E2E tests run against `@copilotkit/aimock` (`LLMock`) — a local HTTP server th
44

55
Before adding an e2e test, check whether the regression can be proven with a package-local unit or integration test. E2E tests should cover real extension-host boundaries and full workflow smoke checks, not detailed assertions that belong to service, protocol, or UI component tests.
66

7+
## Extension Host Visual Tests
8+
9+
The Playwright Electron suite under `src/visual/` is a small production-parity layer, not a replacement for the broader `webview-ui` story-gallery snapshots.
10+
11+
- Keep the Electron matrix limited to high-value shell integration scenes that require the real VS Code host.
12+
- Run it with `xvfb-run -a pnpm --filter @roo-code/vscode-e2e test:visual:electron`.
13+
- Scene setup belongs in `sceneController.ts` and must use the exported Zoo Code test API plus explicit readiness files. Do not gate screenshots with fixed sleeps.
14+
- Every run uses isolated workspace, user-data, and extensions directories and must clean them up on success and failure.
15+
- Identify the Zoo Code webview by its extension ID and frame ancestry, not generated frame IDs or names.
16+
- Keep window geometry, theme, locale, fonts, provider responses, pointer position, and focus deterministic.
17+
- Commit only baselines generated by the pinned CI/Xvfb runner. Failure artifacts include the Playwright report and expected/actual/diff images.
18+
- Treat `_electron` compatibility as part of each pinned VS Code/Playwright upgrade because the API is experimental and depends on Electron fuse support.
19+
720
## How aimock matching works
821

922
Fixtures are matched by **substring**: `incoming_last_user_message.includes(fixture.match.userMessage)`. A fixture fires if its match string appears _anywhere_ in the last user message of the API request.

‎apps/vscode-e2e/package.json‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,12 @@
1313
"themes:update": "pnpm themes:build && node ./out/theme-fixtures/generate.js update",
1414
"themes:check": "pnpm themes:build && node ./out/theme-fixtures/generate.js check",
1515
"themes:test": "pnpm --filter @roo-code/types build && rimraf out && tsc -p tsconfig.json && node --test ./out/theme-fixtures/fixtures.test.js",
16+
"visual:build": "pnpm themes:build",
17+
"test:visual:electron": "pnpm visual:build && playwright test -c playwright-electron.config.ts",
1618
"clean": "rimraf out .turbo"
1719
},
1820
"devDependencies": {
21+
"@playwright/test": "1.62.1",
1922
"@roo-code/config-eslint": "workspace:^",
2023
"@roo-code/config-typescript": "workspace:^",
2124
"@roo-code/types": "workspace:^",
Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
import path from "path"
2+
3+
import { defineConfig } from "@playwright/test"
4+
5+
export default defineConfig({
6+
testDir: "./src/visual",
7+
testMatch: "electron.visual.ts",
8+
outputDir: path.resolve(__dirname, "test-results-electron"),
9+
snapshotPathTemplate: "{testDir}/__screenshots__/{arg}{ext}",
10+
fullyParallel: false,
11+
workers: 1,
12+
repeatEach: process.env.CI ? 3 : 1,
13+
timeout: 3 * 60 * 1_000,
14+
expect: {
15+
timeout: 60_000,
16+
toHaveScreenshot: {
17+
animations: "disabled",
18+
caret: "hide",
19+
},
20+
},
21+
use: {
22+
trace: "retain-on-failure",
23+
},
24+
reporter: process.env.CI
25+
? [
26+
["html", { open: "never", outputFolder: path.resolve(__dirname, "playwright-report-electron") }],
27+
["github"],
28+
["list"],
29+
]
30+
: [["html", { open: "never", outputFolder: path.resolve(__dirname, "playwright-report-electron") }], ["list"]],
31+
})
30.1 KB
Loading
23.7 KB
Loading
21.4 KB
Loading
21.5 KB
Loading
Lines changed: 255 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,255 @@
1+
import fs from "fs/promises"
2+
import { readFileSync } from "fs"
3+
import os from "os"
4+
import path from "path"
5+
6+
import { LLMock } from "@copilotkit/aimock"
7+
import { downloadAndUnzipVSCode } from "@vscode/test-electron"
8+
import { _electron, expect, test, type ElectronApplication, type Page, type TestInfo } from "@playwright/test"
9+
import { glob } from "glob"
10+
11+
const packageRoot = path.resolve(__dirname, "../..")
12+
const repositoryRoot = path.resolve(packageRoot, "../..")
13+
const extensionDevelopmentPath = path.join(repositoryRoot, "src")
14+
const extensionTestsPath = path.join(packageRoot, "out", "visual", "sceneController")
15+
const packageJson = JSON.parse(readFileSync(path.join(packageRoot, "package.json"), "utf8")) as {
16+
devDependencies: Record<string, string>
17+
}
18+
const vscodeVersion = packageJson.devDependencies["@types/vscode"]
19+
20+
const CHAT_PROMPT = "electron-visual-smoke: render a deterministic completion"
21+
const CHAT_RESULT = "Electron visual smoke response rendered inside the real VS Code webview."
22+
23+
interface VisualScenario {
24+
name: string
25+
scene: "welcome" | "chat"
26+
themeId: "Default Dark Modern" | "Default High Contrast"
27+
landmark: string
28+
webviewSnapshot?: string
29+
startMock?: boolean
30+
}
31+
32+
const scenarios: VisualScenario[] = [
33+
{
34+
name: "welcome-dark",
35+
scene: "welcome",
36+
themeId: "Default Dark Modern",
37+
landmark: "Welcome to Zoo Code!",
38+
webviewSnapshot: "electron-welcome-dark-webview.png",
39+
},
40+
{
41+
name: "welcome-high-contrast",
42+
scene: "welcome",
43+
themeId: "Default High Contrast",
44+
landmark: "Welcome to Zoo Code!",
45+
},
46+
{
47+
name: "chat-dark",
48+
scene: "chat",
49+
themeId: "Default Dark Modern",
50+
landmark: CHAT_RESULT,
51+
startMock: true,
52+
},
53+
]
54+
55+
interface RunningScene {
56+
app: ElectronApplication
57+
donePath: string
58+
logPath: string
59+
mock?: InstanceType<typeof LLMock>
60+
page: Page
61+
temporaryRoot: string
62+
userDataPath: string
63+
}
64+
65+
async function waitForJson(filePath: string): Promise<{ landmark: string; scene: string; themeId: string }> {
66+
const deadline = Date.now() + 90_000
67+
while (Date.now() < deadline) {
68+
try {
69+
return JSON.parse(await fs.readFile(filePath, "utf8")) as {
70+
landmark: string
71+
scene: string
72+
themeId: string
73+
}
74+
} catch (error) {
75+
if ((error as NodeJS.ErrnoException).code !== "ENOENT" && !(error instanceof SyntaxError)) throw error
76+
await new Promise((resolve) => setTimeout(resolve, 100))
77+
}
78+
}
79+
throw new Error(`Timed out waiting for Electron visual readiness file: ${filePath}`)
80+
}
81+
82+
async function attachLogs(userDataPath: string, logPath: string, testInfo: TestInfo) {
83+
const sceneLog = await fs.readFile(logPath, "utf8").catch(() => "")
84+
if (sceneLog) await testInfo.attach("scene.log", { body: sceneLog, contentType: "text/plain" })
85+
86+
for (const pattern of ["logs/**/exthost.log", "logs/**/renderer.log", "logs/**/*Zoo-Code.log"]) {
87+
const logs = await glob(pattern, { cwd: userDataPath })
88+
const latest = logs.at(-1)
89+
if (!latest) continue
90+
const contents = await fs.readFile(path.join(userDataPath, latest), "utf8").catch(() => "")
91+
if (contents) await testInfo.attach(path.basename(latest), { body: contents, contentType: "text/plain" })
92+
}
93+
}
94+
95+
async function stopTracing(app: ElectronApplication, testInfo: TestInfo) {
96+
await app
97+
.context()
98+
.tracing.stop({ path: testInfo.outputPath("trace.zip") })
99+
.catch(() => undefined)
100+
}
101+
102+
async function startScene(
103+
vscodeExecutablePath: string,
104+
scenario: VisualScenario,
105+
testInfo: TestInfo,
106+
): Promise<RunningScene> {
107+
const temporaryRoot = await fs.mkdtemp(path.join(os.tmpdir(), "zoo-code-electron-visual-"))
108+
const workspacePath = path.join(temporaryRoot, "workspace")
109+
const userDataPath = path.join(temporaryRoot, "user-data")
110+
const extensionsPath = path.join(temporaryRoot, "extensions")
111+
const readyPath = path.join(temporaryRoot, "ready.json")
112+
const donePath = path.join(temporaryRoot, "done")
113+
const logPath = path.join(temporaryRoot, "scene.log")
114+
await Promise.all([fs.mkdir(workspacePath), fs.mkdir(userDataPath), fs.mkdir(extensionsPath)])
115+
116+
let app: ElectronApplication | undefined
117+
let mock: InstanceType<typeof LLMock> | undefined
118+
let page: Page | undefined
119+
try {
120+
if (scenario.startMock) {
121+
mock = new LLMock({ port: 0 })
122+
mock.addFixture({
123+
match: { userMessage: CHAT_PROMPT },
124+
response: {
125+
toolCalls: [
126+
{
127+
name: "attempt_completion",
128+
arguments: JSON.stringify({ result: CHAT_RESULT }),
129+
id: "call_electron_visual_completion",
130+
},
131+
],
132+
},
133+
})
134+
await mock.start()
135+
}
136+
137+
app = await _electron.launch({
138+
executablePath: vscodeExecutablePath,
139+
args: [
140+
workspacePath,
141+
`--extensionDevelopmentPath=${extensionDevelopmentPath}`,
142+
`--extensionTestsPath=${extensionTestsPath}`,
143+
`--user-data-dir=${userDataPath}`,
144+
`--extensions-dir=${extensionsPath}`,
145+
"--disable-workspace-trust",
146+
"--skip-welcome",
147+
"--skip-release-notes",
148+
"--disable-updates",
149+
"--disable-gpu",
150+
"--force-device-scale-factor=1",
151+
],
152+
env: {
153+
...process.env,
154+
AIMOCK_URL: mock?.url ?? "",
155+
ROO_CODE_VISUAL_CHAT_PROMPT: CHAT_PROMPT,
156+
ROO_CODE_VISUAL_CHAT_RESULT: CHAT_RESULT,
157+
ROO_CODE_VISUAL_DONE_PATH: donePath,
158+
ROO_CODE_VISUAL_LOG_PATH: logPath,
159+
ROO_CODE_VISUAL_READY_PATH: readyPath,
160+
ROO_CODE_VISUAL_SCENE: scenario.scene,
161+
ROO_CODE_VISUAL_THEME: scenario.themeId,
162+
ROO_CODE_THEME_FIXTURE_PROBE: "1",
163+
},
164+
locale: "en-US",
165+
timeout: 60_000,
166+
timezoneId: "UTC",
167+
})
168+
await app.context().tracing.start({ screenshots: true, snapshots: true, sources: true })
169+
page = await app.firstWindow({ timeout: 60_000 })
170+
await app.evaluate(({ BrowserWindow }) => {
171+
const window = BrowserWindow.getAllWindows()[0]
172+
window?.setBounds({ x: 0, y: 0, width: 1200, height: 800 })
173+
window?.focus()
174+
})
175+
const ready = await waitForJson(readyPath)
176+
expect(ready).toMatchObject({ landmark: scenario.landmark, scene: scenario.scene, themeId: scenario.themeId })
177+
return { app, donePath, logPath, mock, page, temporaryRoot, userDataPath }
178+
} catch (error) {
179+
if (page) await page.screenshot({ path: testInfo.outputPath("workbench.png") }).catch(() => undefined)
180+
await attachLogs(userDataPath, logPath, testInfo)
181+
await fs.writeFile(donePath, "done\n", "utf8").catch(() => undefined)
182+
if (app) await stopTracing(app, testInfo)
183+
await app?.close().catch(() => undefined)
184+
await mock?.stop().catch(() => undefined)
185+
await fs.rm(temporaryRoot, { recursive: true, force: true })
186+
const message = error instanceof Error ? error.message : String(error)
187+
throw new Error(message)
188+
}
189+
}
190+
191+
async function attachFailureArtifacts(scene: RunningScene, testInfo: TestInfo) {
192+
await scene.page.screenshot({ path: testInfo.outputPath("workbench.png") }).catch(() => undefined)
193+
await attachLogs(scene.userDataPath, scene.logPath, testInfo)
194+
}
195+
196+
async function stopScene(scene: RunningScene, testInfo: TestInfo) {
197+
await fs.writeFile(scene.donePath, "done\n", "utf8").catch(() => undefined)
198+
await stopTracing(scene.app, testInfo)
199+
await scene.app.close().catch(() => undefined)
200+
await scene.mock?.stop().catch(() => undefined)
201+
await fs.rm(scene.temporaryRoot, { recursive: true, force: true })
202+
}
203+
204+
test.describe.configure({ mode: "serial" })
205+
206+
let vscodeExecutablePath: string
207+
test.beforeAll(async () => {
208+
vscodeExecutablePath = await downloadAndUnzipVSCode({ version: vscodeVersion, extensionDevelopmentPath })
209+
})
210+
211+
for (const scenario of scenarios) {
212+
test(`renders ${scenario.name} in the real VS Code Extension Host`, async ({
213+
browserName: _browserName,
214+
}, testInfo) => {
215+
const running = await startScene(vscodeExecutablePath, scenario, testInfo)
216+
try {
217+
await running.page.mouse.move(900, 400)
218+
await expect
219+
.poll(() =>
220+
running.page
221+
.frames()
222+
.some((frame) => frame.url().includes("extensionId=ZooCodeOrganization.zoo-code")),
223+
)
224+
.toBe(true)
225+
const parentFrame = running.page
226+
.frames()
227+
.find((frame) => frame.url().includes("extensionId=ZooCodeOrganization.zoo-code"))
228+
const contentFrame = parentFrame?.childFrames().find((frame) => frame.url().includes("/fake.html"))
229+
if (!contentFrame) throw new Error("Zoo Code webview content frame not found")
230+
await contentFrame
231+
.getByText(scenario.landmark, { exact: false })
232+
.waitFor({ state: "visible", timeout: 60_000 })
233+
await contentFrame.evaluate(() => {
234+
if (document.activeElement instanceof HTMLElement) document.activeElement.blur()
235+
})
236+
await expect.poll(() => contentFrame.evaluate(() => document.activeElement === document.body)).toBe(true)
237+
await contentFrame.evaluate(() => document.fonts.ready)
238+
239+
const sidebar = running.page.locator(".part.sidebar")
240+
await expect(sidebar).toBeVisible()
241+
await expect(sidebar).toHaveScreenshot(`electron-${scenario.name}-sidebar.png`)
242+
243+
if (scenario.webviewSnapshot) {
244+
const webview = running.page.locator('iframe[src*="extensionId=ZooCodeOrganization.zoo-code"]')
245+
await expect(webview).toHaveCount(1)
246+
await expect(webview).toHaveScreenshot(scenario.webviewSnapshot)
247+
}
248+
} catch (error) {
249+
await attachFailureArtifacts(running, testInfo)
250+
throw error
251+
} finally {
252+
await stopScene(running, testInfo)
253+
}
254+
})
255+
}

0 commit comments

Comments
 (0)