|
| 1 | +import fs from "fs/promises" |
| 2 | +import { readFileSync } from "fs" |
| 3 | +import os from "os" |
| 4 | +import path from "path" |
| 5 | + |
| 6 | +import { LLMock } from "@copilotkit/aimock" |
| 7 | +import { downloadAndUnzipVSCode } from "@vscode/test-electron" |
| 8 | +import { _electron, expect, test, type ElectronApplication, type Page, type TestInfo } from "@playwright/test" |
| 9 | +import { glob } from "glob" |
| 10 | + |
| 11 | +const packageRoot = path.resolve(__dirname, "../..") |
| 12 | +const repositoryRoot = path.resolve(packageRoot, "../..") |
| 13 | +const extensionDevelopmentPath = path.join(repositoryRoot, "src") |
| 14 | +const extensionTestsPath = path.join(packageRoot, "out", "visual", "sceneController") |
| 15 | +const packageJson = JSON.parse(readFileSync(path.join(packageRoot, "package.json"), "utf8")) as { |
| 16 | + devDependencies: Record<string, string> |
| 17 | +} |
| 18 | +const vscodeVersion = packageJson.devDependencies["@types/vscode"] |
| 19 | + |
| 20 | +const CHAT_PROMPT = "electron-visual-smoke: render a deterministic completion" |
| 21 | +const CHAT_RESULT = "Electron visual smoke response rendered inside the real VS Code webview." |
| 22 | + |
| 23 | +interface VisualScenario { |
| 24 | + name: string |
| 25 | + scene: "welcome" | "chat" |
| 26 | + themeId: "Default Dark Modern" | "Default High Contrast" |
| 27 | + landmark: string |
| 28 | + webviewSnapshot?: string |
| 29 | + startMock?: boolean |
| 30 | +} |
| 31 | + |
| 32 | +const scenarios: VisualScenario[] = [ |
| 33 | + { |
| 34 | + name: "welcome-dark", |
| 35 | + scene: "welcome", |
| 36 | + themeId: "Default Dark Modern", |
| 37 | + landmark: "Welcome to Zoo Code!", |
| 38 | + webviewSnapshot: "electron-welcome-dark-webview.png", |
| 39 | + }, |
| 40 | + { |
| 41 | + name: "welcome-high-contrast", |
| 42 | + scene: "welcome", |
| 43 | + themeId: "Default High Contrast", |
| 44 | + landmark: "Welcome to Zoo Code!", |
| 45 | + }, |
| 46 | + { |
| 47 | + name: "chat-dark", |
| 48 | + scene: "chat", |
| 49 | + themeId: "Default Dark Modern", |
| 50 | + landmark: CHAT_RESULT, |
| 51 | + startMock: true, |
| 52 | + }, |
| 53 | +] |
| 54 | + |
| 55 | +interface RunningScene { |
| 56 | + app: ElectronApplication |
| 57 | + donePath: string |
| 58 | + logPath: string |
| 59 | + mock?: InstanceType<typeof LLMock> |
| 60 | + page: Page |
| 61 | + temporaryRoot: string |
| 62 | + userDataPath: string |
| 63 | +} |
| 64 | + |
| 65 | +async function waitForJson(filePath: string): Promise<{ landmark: string; scene: string; themeId: string }> { |
| 66 | + const deadline = Date.now() + 90_000 |
| 67 | + while (Date.now() < deadline) { |
| 68 | + try { |
| 69 | + return JSON.parse(await fs.readFile(filePath, "utf8")) as { |
| 70 | + landmark: string |
| 71 | + scene: string |
| 72 | + themeId: string |
| 73 | + } |
| 74 | + } catch (error) { |
| 75 | + if ((error as NodeJS.ErrnoException).code !== "ENOENT" && !(error instanceof SyntaxError)) throw error |
| 76 | + await new Promise((resolve) => setTimeout(resolve, 100)) |
| 77 | + } |
| 78 | + } |
| 79 | + throw new Error(`Timed out waiting for Electron visual readiness file: ${filePath}`) |
| 80 | +} |
| 81 | + |
| 82 | +async function attachLogs(userDataPath: string, logPath: string, testInfo: TestInfo) { |
| 83 | + const sceneLog = await fs.readFile(logPath, "utf8").catch(() => "") |
| 84 | + if (sceneLog) await testInfo.attach("scene.log", { body: sceneLog, contentType: "text/plain" }) |
| 85 | + |
| 86 | + for (const pattern of ["logs/**/exthost.log", "logs/**/renderer.log", "logs/**/*Zoo-Code.log"]) { |
| 87 | + const logs = await glob(pattern, { cwd: userDataPath }) |
| 88 | + const latest = logs.at(-1) |
| 89 | + if (!latest) continue |
| 90 | + const contents = await fs.readFile(path.join(userDataPath, latest), "utf8").catch(() => "") |
| 91 | + if (contents) await testInfo.attach(path.basename(latest), { body: contents, contentType: "text/plain" }) |
| 92 | + } |
| 93 | +} |
| 94 | + |
| 95 | +async function stopTracing(app: ElectronApplication, testInfo: TestInfo) { |
| 96 | + await app |
| 97 | + .context() |
| 98 | + .tracing.stop({ path: testInfo.outputPath("trace.zip") }) |
| 99 | + .catch(() => undefined) |
| 100 | +} |
| 101 | + |
| 102 | +async function startScene( |
| 103 | + vscodeExecutablePath: string, |
| 104 | + scenario: VisualScenario, |
| 105 | + testInfo: TestInfo, |
| 106 | +): Promise<RunningScene> { |
| 107 | + const temporaryRoot = await fs.mkdtemp(path.join(os.tmpdir(), "zoo-code-electron-visual-")) |
| 108 | + const workspacePath = path.join(temporaryRoot, "workspace") |
| 109 | + const userDataPath = path.join(temporaryRoot, "user-data") |
| 110 | + const extensionsPath = path.join(temporaryRoot, "extensions") |
| 111 | + const readyPath = path.join(temporaryRoot, "ready.json") |
| 112 | + const donePath = path.join(temporaryRoot, "done") |
| 113 | + const logPath = path.join(temporaryRoot, "scene.log") |
| 114 | + await Promise.all([fs.mkdir(workspacePath), fs.mkdir(userDataPath), fs.mkdir(extensionsPath)]) |
| 115 | + |
| 116 | + let app: ElectronApplication | undefined |
| 117 | + let mock: InstanceType<typeof LLMock> | undefined |
| 118 | + let page: Page | undefined |
| 119 | + try { |
| 120 | + if (scenario.startMock) { |
| 121 | + mock = new LLMock({ port: 0 }) |
| 122 | + mock.addFixture({ |
| 123 | + match: { userMessage: CHAT_PROMPT }, |
| 124 | + response: { |
| 125 | + toolCalls: [ |
| 126 | + { |
| 127 | + name: "attempt_completion", |
| 128 | + arguments: JSON.stringify({ result: CHAT_RESULT }), |
| 129 | + id: "call_electron_visual_completion", |
| 130 | + }, |
| 131 | + ], |
| 132 | + }, |
| 133 | + }) |
| 134 | + await mock.start() |
| 135 | + } |
| 136 | + |
| 137 | + app = await _electron.launch({ |
| 138 | + executablePath: vscodeExecutablePath, |
| 139 | + args: [ |
| 140 | + workspacePath, |
| 141 | + `--extensionDevelopmentPath=${extensionDevelopmentPath}`, |
| 142 | + `--extensionTestsPath=${extensionTestsPath}`, |
| 143 | + `--user-data-dir=${userDataPath}`, |
| 144 | + `--extensions-dir=${extensionsPath}`, |
| 145 | + "--disable-workspace-trust", |
| 146 | + "--skip-welcome", |
| 147 | + "--skip-release-notes", |
| 148 | + "--disable-updates", |
| 149 | + "--disable-gpu", |
| 150 | + "--force-device-scale-factor=1", |
| 151 | + ], |
| 152 | + env: { |
| 153 | + ...process.env, |
| 154 | + AIMOCK_URL: mock?.url ?? "", |
| 155 | + ROO_CODE_VISUAL_CHAT_PROMPT: CHAT_PROMPT, |
| 156 | + ROO_CODE_VISUAL_CHAT_RESULT: CHAT_RESULT, |
| 157 | + ROO_CODE_VISUAL_DONE_PATH: donePath, |
| 158 | + ROO_CODE_VISUAL_LOG_PATH: logPath, |
| 159 | + ROO_CODE_VISUAL_READY_PATH: readyPath, |
| 160 | + ROO_CODE_VISUAL_SCENE: scenario.scene, |
| 161 | + ROO_CODE_VISUAL_THEME: scenario.themeId, |
| 162 | + ROO_CODE_THEME_FIXTURE_PROBE: "1", |
| 163 | + }, |
| 164 | + locale: "en-US", |
| 165 | + timeout: 60_000, |
| 166 | + timezoneId: "UTC", |
| 167 | + }) |
| 168 | + await app.context().tracing.start({ screenshots: true, snapshots: true, sources: true }) |
| 169 | + page = await app.firstWindow({ timeout: 60_000 }) |
| 170 | + await app.evaluate(({ BrowserWindow }) => { |
| 171 | + const window = BrowserWindow.getAllWindows()[0] |
| 172 | + window?.setBounds({ x: 0, y: 0, width: 1200, height: 800 }) |
| 173 | + window?.focus() |
| 174 | + }) |
| 175 | + const ready = await waitForJson(readyPath) |
| 176 | + expect(ready).toMatchObject({ landmark: scenario.landmark, scene: scenario.scene, themeId: scenario.themeId }) |
| 177 | + return { app, donePath, logPath, mock, page, temporaryRoot, userDataPath } |
| 178 | + } catch (error) { |
| 179 | + if (page) await page.screenshot({ path: testInfo.outputPath("workbench.png") }).catch(() => undefined) |
| 180 | + await attachLogs(userDataPath, logPath, testInfo) |
| 181 | + await fs.writeFile(donePath, "done\n", "utf8").catch(() => undefined) |
| 182 | + if (app) await stopTracing(app, testInfo) |
| 183 | + await app?.close().catch(() => undefined) |
| 184 | + await mock?.stop().catch(() => undefined) |
| 185 | + await fs.rm(temporaryRoot, { recursive: true, force: true }) |
| 186 | + const message = error instanceof Error ? error.message : String(error) |
| 187 | + throw new Error(message) |
| 188 | + } |
| 189 | +} |
| 190 | + |
| 191 | +async function attachFailureArtifacts(scene: RunningScene, testInfo: TestInfo) { |
| 192 | + await scene.page.screenshot({ path: testInfo.outputPath("workbench.png") }).catch(() => undefined) |
| 193 | + await attachLogs(scene.userDataPath, scene.logPath, testInfo) |
| 194 | +} |
| 195 | + |
| 196 | +async function stopScene(scene: RunningScene, testInfo: TestInfo) { |
| 197 | + await fs.writeFile(scene.donePath, "done\n", "utf8").catch(() => undefined) |
| 198 | + await stopTracing(scene.app, testInfo) |
| 199 | + await scene.app.close().catch(() => undefined) |
| 200 | + await scene.mock?.stop().catch(() => undefined) |
| 201 | + await fs.rm(scene.temporaryRoot, { recursive: true, force: true }) |
| 202 | +} |
| 203 | + |
| 204 | +test.describe.configure({ mode: "serial" }) |
| 205 | + |
| 206 | +let vscodeExecutablePath: string |
| 207 | +test.beforeAll(async () => { |
| 208 | + vscodeExecutablePath = await downloadAndUnzipVSCode({ version: vscodeVersion, extensionDevelopmentPath }) |
| 209 | +}) |
| 210 | + |
| 211 | +for (const scenario of scenarios) { |
| 212 | + test(`renders ${scenario.name} in the real VS Code Extension Host`, async ({ |
| 213 | + browserName: _browserName, |
| 214 | + }, testInfo) => { |
| 215 | + const running = await startScene(vscodeExecutablePath, scenario, testInfo) |
| 216 | + try { |
| 217 | + await running.page.mouse.move(900, 400) |
| 218 | + await expect |
| 219 | + .poll(() => |
| 220 | + running.page |
| 221 | + .frames() |
| 222 | + .some((frame) => frame.url().includes("extensionId=ZooCodeOrganization.zoo-code")), |
| 223 | + ) |
| 224 | + .toBe(true) |
| 225 | + const parentFrame = running.page |
| 226 | + .frames() |
| 227 | + .find((frame) => frame.url().includes("extensionId=ZooCodeOrganization.zoo-code")) |
| 228 | + const contentFrame = parentFrame?.childFrames().find((frame) => frame.url().includes("/fake.html")) |
| 229 | + if (!contentFrame) throw new Error("Zoo Code webview content frame not found") |
| 230 | + await contentFrame |
| 231 | + .getByText(scenario.landmark, { exact: false }) |
| 232 | + .waitFor({ state: "visible", timeout: 60_000 }) |
| 233 | + await contentFrame.evaluate(() => { |
| 234 | + if (document.activeElement instanceof HTMLElement) document.activeElement.blur() |
| 235 | + }) |
| 236 | + await expect.poll(() => contentFrame.evaluate(() => document.activeElement === document.body)).toBe(true) |
| 237 | + await contentFrame.evaluate(() => document.fonts.ready) |
| 238 | + |
| 239 | + const sidebar = running.page.locator(".part.sidebar") |
| 240 | + await expect(sidebar).toBeVisible() |
| 241 | + await expect(sidebar).toHaveScreenshot(`electron-${scenario.name}-sidebar.png`) |
| 242 | + |
| 243 | + if (scenario.webviewSnapshot) { |
| 244 | + const webview = running.page.locator('iframe[src*="extensionId=ZooCodeOrganization.zoo-code"]') |
| 245 | + await expect(webview).toHaveCount(1) |
| 246 | + await expect(webview).toHaveScreenshot(scenario.webviewSnapshot) |
| 247 | + } |
| 248 | + } catch (error) { |
| 249 | + await attachFailureArtifacts(running, testInfo) |
| 250 | + throw error |
| 251 | + } finally { |
| 252 | + await stopScene(running, testInfo) |
| 253 | + } |
| 254 | + }) |
| 255 | +} |
0 commit comments