|
| 1 | +import fs from "fs/promises" |
| 2 | +import { closeSync, openSync, readFileSync, writeSync } from "fs" |
| 3 | +import os from "os" |
| 4 | +import path from "path" |
| 5 | +import { LLMock } from "@copilotkit/aimock" |
| 6 | +import { downloadAndUnzipVSCode } from "@vscode/test-electron" |
| 7 | +import { _electron, expect, test, type CDPSession, type ElectronApplication, type Frame } from "@playwright/test" |
| 8 | +import { createTranscript, LARGE_TASK_MARKER, SMALL_TASK_MARKER, type Workload } from "./workload" |
| 9 | + |
| 10 | +const packageRoot = path.resolve(__dirname, "../..") |
| 11 | +const results = process.env.HEAP_BENCHMARK_DIR ?? path.join(packageRoot, "heap-results") |
| 12 | +const packageJson = JSON.parse(readFileSync(path.join(packageRoot, "package.json"), "utf8")) as { |
| 13 | + devDependencies: Record<string, string> |
| 14 | +} |
| 15 | + |
| 16 | +async function waitForFile(file: string): Promise<void> { |
| 17 | + await expect |
| 18 | + .poll( |
| 19 | + () => |
| 20 | + fs.access(file).then( |
| 21 | + () => true, |
| 22 | + () => false, |
| 23 | + ), |
| 24 | + { timeout: 120_000 }, |
| 25 | + ) |
| 26 | + .toBe(true) |
| 27 | +} |
| 28 | + |
| 29 | +async function capture(client: CDPSession, file: string) { |
| 30 | + await client.send("HeapProfiler.collectGarbage") |
| 31 | + const usage = await client.send("Runtime.getHeapUsage") |
| 32 | + const descriptor = openSync(file, "w") |
| 33 | + const receive = ({ chunk }: { chunk: string }) => { |
| 34 | + writeSync(descriptor, chunk) |
| 35 | + } |
| 36 | + client.on("HeapProfiler.addHeapSnapshotChunk", receive) |
| 37 | + try { |
| 38 | + await client.send("HeapProfiler.takeHeapSnapshot", { reportProgress: false }) |
| 39 | + } finally { |
| 40 | + client.off("HeapProfiler.addHeapSnapshotChunk", receive) |
| 41 | + closeSync(descriptor) |
| 42 | + } |
| 43 | + return usage |
| 44 | +} |
| 45 | + |
| 46 | +for (const workload of ["repeated", "unique"] satisfies Workload[]) { |
| 47 | + test(`webview retained heap: ${workload} tool outputs`, async () => { |
| 48 | + const testInfo = test.info() |
| 49 | + const temporaryRoot = await fs.mkdtemp(path.join(os.tmpdir(), "zoo-heap-")) |
| 50 | + const workspace = path.join(temporaryRoot, "workspace") |
| 51 | + const userData = path.join(temporaryRoot, "user-data") |
| 52 | + const extensions = path.join(temporaryRoot, "extensions") |
| 53 | + await Promise.all([workspace, userData, extensions].map((directory) => fs.mkdir(directory))) |
| 54 | + const mock = new LLMock({ port: 0 }) |
| 55 | + for (const marker of [SMALL_TASK_MARKER, "HEAP_TRANSCRIPT_SEED"]) { |
| 56 | + mock.addFixture({ |
| 57 | + match: { userMessage: marker }, |
| 58 | + response: { |
| 59 | + toolCalls: [ |
| 60 | + { |
| 61 | + name: "attempt_completion", |
| 62 | + arguments: JSON.stringify({ result: `${marker}_COMPLETED` }), |
| 63 | + id: `call_${marker}`, |
| 64 | + }, |
| 65 | + ], |
| 66 | + }, |
| 67 | + }) |
| 68 | + } |
| 69 | + let app: ElectronApplication | undefined |
| 70 | + let client: CDPSession | undefined |
| 71 | + try { |
| 72 | + await mock.start() |
| 73 | + const executablePath = |
| 74 | + process.env.HEAP_VSCODE_EXECUTABLE ?? |
| 75 | + (await downloadAndUnzipVSCode({ version: packageJson.devDependencies["@types/vscode"] })) |
| 76 | + const launch = (phase: "seed" | "measure") => |
| 77 | + _electron.launch({ |
| 78 | + executablePath, |
| 79 | + args: [ |
| 80 | + workspace, |
| 81 | + `--extensionDevelopmentPath=${path.join(results, "builds", testInfo.project.name, "src")}`, |
| 82 | + `--extensionTestsPath=${path.join(packageRoot, "out/heap/sceneController")}`, |
| 83 | + `--user-data-dir=${userData}`, |
| 84 | + `--extensions-dir=${extensions}`, |
| 85 | + "--disable-workspace-trust", |
| 86 | + "--skip-welcome", |
| 87 | + "--skip-release-notes", |
| 88 | + "--disable-updates", |
| 89 | + "--disable-gpu", |
| 90 | + "--force-device-scale-factor=1", |
| 91 | + ], |
| 92 | + env: { |
| 93 | + ...process.env, |
| 94 | + AIMOCK_URL: mock.url, |
| 95 | + HEAP_RUN_DIR: temporaryRoot, |
| 96 | + HEAP_WORKLOAD: workload, |
| 97 | + HEAP_PHASE: phase, |
| 98 | + HTTP_PROXY: "http://127.0.0.1:9", |
| 99 | + HTTPS_PROXY: "http://127.0.0.1:9", |
| 100 | + ALL_PROXY: "http://127.0.0.1:9", |
| 101 | + NO_PROXY: "127.0.0.1,localhost", |
| 102 | + }, |
| 103 | + locale: "en-US", |
| 104 | + timezoneId: "UTC", |
| 105 | + timeout: 60_000, |
| 106 | + }) |
| 107 | + |
| 108 | + app = await launch("seed") |
| 109 | + await waitForFile(path.join(temporaryRoot, "seed.ready")) |
| 110 | + await fs.writeFile(path.join(temporaryRoot, "seed.close"), "close") |
| 111 | + await app.close() |
| 112 | + app = undefined |
| 113 | + // No extension host is alive while replacing the persisted transcript. |
| 114 | + const seed = JSON.parse(await fs.readFile(path.join(temporaryRoot, "seed.ready"), "utf8")) as { |
| 115 | + largeTaskId: string |
| 116 | + } |
| 117 | + await fs.writeFile( |
| 118 | + path.join(temporaryRoot, "storage/tasks", seed.largeTaskId, "ui_messages.json"), |
| 119 | + JSON.stringify(createTranscript(workload)), |
| 120 | + ) |
| 121 | + app = await launch("measure") |
| 122 | + const page = await app.firstWindow() |
| 123 | + await app.evaluate(({ BrowserWindow }) => |
| 124 | + BrowserWindow.getAllWindows()[0]?.setBounds({ x: 0, y: 0, width: 1200, height: 800 }), |
| 125 | + ) |
| 126 | + await waitForFile(path.join(temporaryRoot, "empty.ready")) |
| 127 | + let content: Frame | undefined |
| 128 | + await expect |
| 129 | + .poll(() => { |
| 130 | + const parent = page |
| 131 | + .frames() |
| 132 | + .find((frame) => frame.url().includes("extensionId=ZooCodeOrganization.zoo-code")) |
| 133 | + content = parent?.childFrames().find((frame) => frame.url().includes("/fake.html")) |
| 134 | + return Boolean(content) |
| 135 | + }) |
| 136 | + .toBe(true) |
| 137 | + if (!content) throw new Error("Zoo Code webview content frame not found") |
| 138 | + // The content iframe may share its CDP target with the outer webview. |
| 139 | + let targetFrame: Frame | null = content |
| 140 | + while (targetFrame) { |
| 141 | + try { |
| 142 | + client = await app.context().newCDPSession(targetFrame) |
| 143 | + break |
| 144 | + } catch (error) { |
| 145 | + if (!(error instanceof Error) || !error.message.includes("does not have a separate CDP session")) |
| 146 | + throw error |
| 147 | + targetFrame = targetFrame.parentFrame() |
| 148 | + } |
| 149 | + } |
| 150 | + if (!client) throw new Error("No CDP target found for the webview") |
| 151 | + await client.send("HeapProfiler.enable") |
| 152 | + const target = await client.send("Target.getTargetInfo") |
| 153 | + expect(target.targetInfo.url).toContain("vscode-webview") |
| 154 | + const stages: Record<string, unknown> = {} |
| 155 | + for (const stage of ["empty", "loaded", "switched"]) { |
| 156 | + await waitForFile(path.join(temporaryRoot, `${stage}.ready`)) |
| 157 | + if (stage !== "empty") { |
| 158 | + await expect( |
| 159 | + content |
| 160 | + .getByText(stage === "loaded" ? LARGE_TASK_MARKER : `${SMALL_TASK_MARKER}_COMPLETED`, { |
| 161 | + exact: false, |
| 162 | + }) |
| 163 | + .first(), |
| 164 | + ).toBeVisible() |
| 165 | + } |
| 166 | + await content.evaluate(async () => { |
| 167 | + await document.fonts.ready |
| 168 | + await new Promise<void>((resolve) => |
| 169 | + requestAnimationFrame(() => requestAnimationFrame(() => resolve())), |
| 170 | + ) |
| 171 | + }) |
| 172 | + stages[stage] = await capture(client, testInfo.outputPath(`${stage}.heapsnapshot`)) |
| 173 | + await fs.writeFile(path.join(temporaryRoot, `${stage}.continue`), "continue") |
| 174 | + } |
| 175 | + await waitForFile(path.join(temporaryRoot, "done")) |
| 176 | + const measurement = { |
| 177 | + variant: testInfo.project.name, |
| 178 | + workload, |
| 179 | + repeat: testInfo.repeatEachIndex, |
| 180 | + vscode: packageJson.devDependencies["@types/vscode"], |
| 181 | + target, |
| 182 | + stages, |
| 183 | + } |
| 184 | + await fs.writeFile(testInfo.outputPath("measurement.json"), JSON.stringify(measurement, null, 2)) |
| 185 | + await testInfo.attach("measurement", { body: JSON.stringify(measurement), contentType: "application/json" }) |
| 186 | + } catch (error) { |
| 187 | + if (app) |
| 188 | + await ( |
| 189 | + await app.firstWindow() |
| 190 | + ) |
| 191 | + .screenshot({ path: testInfo.outputPath("failure.png") }) |
| 192 | + .catch(() => undefined) |
| 193 | + await fs |
| 194 | + .cp(path.join(userData, "logs"), testInfo.outputPath("logs"), { recursive: true }) |
| 195 | + .catch(() => undefined) |
| 196 | + throw error |
| 197 | + } finally { |
| 198 | + await fs.writeFile(path.join(temporaryRoot, "close"), "close").catch(() => undefined) |
| 199 | + await client?.detach().catch(() => undefined) |
| 200 | + await app?.close().catch(() => undefined) |
| 201 | + await mock.stop().catch(() => undefined) |
| 202 | + await fs.rm(temporaryRoot, { recursive: true, force: true }) |
| 203 | + } |
| 204 | + }) |
| 205 | +} |
0 commit comments