Skip to content

Commit 4124c43

Browse files
Merge remote-tracking branch 'upstream/main' into HEAD
2 parents 1b9e8f8 + 159c42e commit 4124c43

104 files changed

Lines changed: 8541 additions & 1107 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ AGENTS.local.md
2727
.vscode-test/
2828
apps/vscode-e2e/playwright-report-electron/
2929
apps/vscode-e2e/test-results-electron/
30+
apps/vscode-e2e/heap-results/
3031

3132
# Docs
3233
docs/_site/

‎apps/vscode-e2e/AGENTS.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -278,3 +278,7 @@ Always restore the default OpenRouter config in `suiteTeardown` so subsequent su
278278
## Programmatic fixtures (regex matching)
279279
280280
For requests that can't be matched by a stable substring (e.g. "starts with `<environment_details>` but not preceded by a user message"), add a programmatic fixture in `src/runTest.ts` using `mock.addFixture()` with a `RegExp` match. These are only available in replay mode and are not recorded.
281+
282+
## Webview heap diagnostic
283+
284+
The opt-in suite under `src/heap/` measures retained renderer heap through real VS Code task hydration and switching. See [src/heap/README.md](src/heap/README.md) for build, run, and summarize commands. Keep base and candidate host builds identical, use isolated profiles, and preserve cleanup on failure. Heap snapshots and generated measurements stay in the git-ignored `heap-results/` directory. This diagnostic has no CI memory threshold and does not replace focused webview tests.

‎apps/vscode-e2e/package.json‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,10 @@
1515
"themes:test": "pnpm --filter @roo-code/types build && rimraf out && tsc -p tsconfig.json && node --test ./out/theme-fixtures/fixtures.test.js",
1616
"visual:build": "pnpm themes:build",
1717
"test:visual:electron": "pnpm visual:build && playwright test -c playwright-electron.config.ts",
18-
"clean": "rimraf out .turbo"
18+
"clean": "rimraf out .turbo",
19+
"heap:build": "pnpm -w bundle && tsc -p tsconfig.json && node out/heap/prepare.js",
20+
"test:heap": "playwright test -c playwright-heap.config.ts",
21+
"heap:summarize": "node out/heap/summarize.js"
1922
},
2023
"devDependencies": {
2124
"@playwright/test": "1.62.1",
Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
import path from "path"
2+
import { defineConfig } from "@playwright/test"
3+
4+
const results = process.env.HEAP_BENCHMARK_DIR ?? path.resolve(__dirname, "heap-results")
5+
export default defineConfig({
6+
testDir: "./src/heap",
7+
testMatch: "electron.heap.ts",
8+
outputDir: path.join(results, "runs"),
9+
fullyParallel: false,
10+
workers: 1,
11+
repeatEach: Number(process.env.HEAP_REPEATS ?? 3),
12+
timeout: 5 * 60_000,
13+
expect: { timeout: 90_000 },
14+
projects: [{ name: "base" }, { name: "candidate" }],
15+
reporter: [["list"], ["json", { outputFile: path.join(results, "playwright.json") }]],
16+
})

‎apps/vscode-e2e/src/heap/README.md‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
# Webview heap benchmark
2+
3+
See [RESULTS.md](./RESULTS.md) for the recorded comparison and its retention findings.
4+
5+
This opt-in diagnostic runs real VS Code, the extension host, task-history hydration,
6+
extension/webview messaging and the production React webview. The model is a local
7+
mock. It does not call a paid API or mutate your normal VS Code profile.
8+
9+
```sh
10+
pnpm --dir apps/vscode-e2e heap:build
11+
xvfb-run -a pnpm --dir apps/vscode-e2e test:heap
12+
pnpm --dir apps/vscode-e2e heap:summarize
13+
```
14+
15+
`heap:build` creates isolated base and candidate webview builds in `heap-results/builds`.
16+
The extension host is identical for both. Base replaces `webview-ui/src/context/ExtensionStateContext.tsx`
17+
with its version at `git merge-base HEAD origin/main`.
18+
Override that reference with `HEAP_BASE_REF`. Source maps are disabled for both builds
19+
to reduce build memory; production minification is preserved. Source files in your
20+
checkout are untouched.
21+
22+
Each repetition creates two tasks through the exported API, evicts them, and stops
23+
VS Code. With no extension host alive, it replaces the large task's persisted UI
24+
transcript with 602 deterministic messages:
25+
300 completed command/output pairs plus a title and completion. Each output contains
26+
256 log lines (roughly 17 KiB). `repeated` uses 32 distinct output texts; `unique` uses 300. The isolated profile is restarted, and the normal task-resume path reads and posts
27+
the transcript. This represents a
28+
reproducible large saved task, not a recording of a human or live model session.
29+
30+
The driver waits for task-resume events, extension readiness files and rendered
31+
completion markers. It requests a normal full-state refresh after each resume. It
32+
then waits for fonts and two animation frames, forces GC through CDP attached to the
33+
Zoo Code content frame, records `Runtime.getHeapUsage`, and streams a heap snapshot
34+
to disk. This happens with no task loaded, with the large task loaded, and after
35+
switching to a small task. A final snapshot can expose strings retained after the
36+
task transition. This measures the webview renderer isolate, not extension-host heap
37+
or whole-process RSS. External HTTP traffic is blocked (only the local mock model is reachable).
38+
Playwright tracing is disabled to avoid retaining captured DOM
39+
snapshots and message text during measurement.
40+
41+
The default is three fresh sessions per workload and variant (12 sessions). There is
42+
no pass/fail memory threshold. Inspect distributions, baseline-adjusted differences,
43+
and retaining paths before drawing conclusions. The summarizer reports median heap
44+
sizes and counts large output string nodes in the loaded and switched snapshots.
45+
It does not infer cache ownership solely from a surviving string.
46+
47+
Artifacts are ignored by git under `apps/vscode-e2e/heap-results/`: build provenance,
48+
per-run `measurement.json`, three `.heapsnapshot` files, `summary.json` and `summary.md`.
49+
Load snapshots in Chrome DevTools' Memory panel for retaining-path analysis. Temporary
50+
profiles, workspaces and extension directories are removed on both success and failure;
51+
failure logs and screenshots are copied to the run artifacts first.
52+
53+
Optional environment variables (use the same `HEAP_BENCHMARK_DIR` for all three commands):
54+
55+
- `HEAP_BENCHMARK_DIR`: absolute artifact/build directory.
56+
- `HEAP_BASE_REF`: base commit or ref used by the comparison build.
57+
- `HEAP_REPEATS`: repetitions per workload and variant (default 3).
58+
- `HEAP_VSCODE_EXECUTABLE`: existing VS Code executable; otherwise download the pinned version.
59+
60+
For a smoke run: `HEAP_REPEATS=1 ... test:heap --project=candidate --grep=repeated`.
Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
# Webview heap results — 2026-10-27
2+
3+
Measured with real VS Code 1.100.0 under Xvfb, using CDP forced GC on the Zoo Code
4+
webview renderer target. These are deterministic saved-task workloads loaded through
5+
the real extension host and task-resume path, not an organic human/model session.
6+
The extension host, dependencies and styles are identical between variants. Both
7+
webview builds use production minification without source maps. External network
8+
requests are blocked except for the local mock model used to create the seed tasks.
9+
10+
- Base webview modules: `97228c7ca9b032e5fe466e57e24fe72fbf405192`.
11+
- Candidate webview modules: `b9a6ba03cb93c0f92f06cfb00c82ae7d488db7cb`.
12+
- Three fresh measurement sessions per workload and variant: 12 successful samples.
13+
- Each large task has 602 UI messages, including 300 command/output pairs. Repeated
14+
uses 32 distinct large output strings; unique uses 300. Each output is 16.4–16.9 KiB.
15+
A full-state refresh after
16+
hydration exercises reuse across successive deserialized state snapshots.
17+
18+
These hashes identify the historical measured builds, not the current branch head.
19+
The measurements below have not been rerun for subsequent review fixes.
20+
21+
Median forced-GC used heap, in MiB:
22+
23+
| Workload | Variant | Empty | Large task loaded | Loaded − empty | After switching to small task |
24+
| -------- | --------- | ----: | ----------------: | -------------: | ----------------------------: |
25+
| Repeated | Base | 26.46 | 41.42 | 14.71 | 36.19 |
26+
| Repeated | Candidate | 26.57 | 31.75 | 5.19 | 31.85 |
27+
| Unique | Base | 26.78 | 41.58 | 14.80 | 36.27 |
28+
| Unique | Candidate | 26.57 | 36.63 | 10.07 | 36.26 |
29+
30+
`Loaded − empty` is the median of each session’s loaded-minus-empty difference;
31+
it need not equal the difference of the two column medians.
32+
33+
Loaded heap is approximately **9.67 MiB lower (23%)** for repeated outputs and
34+
**4.95 MiB lower (12%)** for unique outputs. Baseline-adjusted task growth falls by
35+
about 9.52 MiB and 4.73 MiB respectively. The unique workload still benefits from
36+
sharing unchanged message strings across full-state pushes via `Object.assign`,
37+
even after FIFO eviction. The cache can retain only about 120 strings of 16 KiB
38+
within its 2M-character budget. Repeated outputs fit within that budget and benefit
39+
from cache interning as well. These mechanisms explain the observed difference,
40+
but this comparison does not isolate their individual contributions. These results do not
41+
measure CPU overhead, image-rich messages, or continuously streamed task execution.
42+
43+
Loaded-heap ranges across the three sessions:
44+
45+
| Workload | Base MiB | Candidate MiB |
46+
| -------- | ----------- | ------------- |
47+
| Repeated | 40.96–41.42 | 31.75–32.23 |
48+
| Unique | 41.14–41.59 | 36.62–36.64 |
49+
50+
## Task-switch retention
51+
52+
The candidate's StringCache maps retain **zero old output strings after switching**
53+
in all six candidate samples. Before switching, the message cache contains 32
54+
output strings in the repeated case and 115 in the unique case, consistent with its
55+
character budget. The history cache contains no tool outputs.
56+
57+
However, old task output strings remain reachable elsewhere in **both** variants:
58+
300 output string nodes on base, versus 32 repeated or 300 unique on candidate.
59+
A shortest strong retaining path inspected in both switched snapshots runs through:
60+
61+
```text
62+
webview Window → DOM node → __reactProps… → children → props
63+
→ onJumpToPreviousCheckpoint → closure contexts → message array → message.text
64+
```
65+
66+
This confirms cache cleanup, but does **not** establish complete task-memory release.
67+
The callback/React-props retaining path needs a separate investigation; this benchmark
68+
only measures the observed post-switch state and does not establish indefinite growth.
69+
No forced remounts or additional product behavior changes were introduced here.
70+
71+
## Evidence and reproduction
72+
73+
See [README.md](./README.md) for commands and methodology. The raw artifacts are local, git-ignored evidence and are not distributed with this
74+
report; reproducing the run is required to independently inspect them. The ignored
75+
`apps/vscode-e2e/heap-results/` directory contains the 36 successful snapshots,
76+
per-run measurements, build provenance, generated summary, cache-map inventory,
77+
and representative base/candidate retaining paths under `analysis/`.
78+
79+
The final sample set consists of 11 successful matrix cases and a successful narrow
80+
rerun of the first base case after fixing the diagnostic's resume-readiness check.
81+
The failed setup attempt has no measurement record and is excluded from the summary;
82+
its failure artifacts remain available. Package type checks and ESLint pass with no
83+
new suppressions. No memory threshold is imposed on CI.
Lines changed: 205 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,205 @@
1+
import fs from "fs/promises"
2+
import { closeSync, openSync, readFileSync, writeSync } from "fs"
3+
import os from "os"
4+
import path from "path"
5+
import { LLMock } from "@copilotkit/aimock"
6+
import { downloadAndUnzipVSCode } from "@vscode/test-electron"
7+
import { _electron, expect, test, type CDPSession, type ElectronApplication, type Frame } from "@playwright/test"
8+
import { createTranscript, LARGE_TASK_MARKER, SMALL_TASK_MARKER, type Workload } from "./workload"
9+
10+
const packageRoot = path.resolve(__dirname, "../..")
11+
const results = process.env.HEAP_BENCHMARK_DIR ?? path.join(packageRoot, "heap-results")
12+
const packageJson = JSON.parse(readFileSync(path.join(packageRoot, "package.json"), "utf8")) as {
13+
devDependencies: Record<string, string>
14+
}
15+
16+
async function waitForFile(file: string): Promise<void> {
17+
await expect
18+
.poll(
19+
() =>
20+
fs.access(file).then(
21+
() => true,
22+
() => false,
23+
),
24+
{ timeout: 120_000 },
25+
)
26+
.toBe(true)
27+
}
28+
29+
async function capture(client: CDPSession, file: string) {
30+
await client.send("HeapProfiler.collectGarbage")
31+
const usage = await client.send("Runtime.getHeapUsage")
32+
const descriptor = openSync(file, "w")
33+
const receive = ({ chunk }: { chunk: string }) => {
34+
writeSync(descriptor, chunk)
35+
}
36+
client.on("HeapProfiler.addHeapSnapshotChunk", receive)
37+
try {
38+
await client.send("HeapProfiler.takeHeapSnapshot", { reportProgress: false })
39+
} finally {
40+
client.off("HeapProfiler.addHeapSnapshotChunk", receive)
41+
closeSync(descriptor)
42+
}
43+
return usage
44+
}
45+
46+
for (const workload of ["repeated", "unique"] satisfies Workload[]) {
47+
test(`webview retained heap: ${workload} tool outputs`, async () => {
48+
const testInfo = test.info()
49+
const temporaryRoot = await fs.mkdtemp(path.join(os.tmpdir(), "zoo-heap-"))
50+
const workspace = path.join(temporaryRoot, "workspace")
51+
const userData = path.join(temporaryRoot, "user-data")
52+
const extensions = path.join(temporaryRoot, "extensions")
53+
await Promise.all([workspace, userData, extensions].map((directory) => fs.mkdir(directory)))
54+
const mock = new LLMock({ port: 0 })
55+
for (const marker of [SMALL_TASK_MARKER, "HEAP_TRANSCRIPT_SEED"]) {
56+
mock.addFixture({
57+
match: { userMessage: marker },
58+
response: {
59+
toolCalls: [
60+
{
61+
name: "attempt_completion",
62+
arguments: JSON.stringify({ result: `${marker}_COMPLETED` }),
63+
id: `call_${marker}`,
64+
},
65+
],
66+
},
67+
})
68+
}
69+
let app: ElectronApplication | undefined
70+
let client: CDPSession | undefined
71+
try {
72+
await mock.start()
73+
const executablePath =
74+
process.env.HEAP_VSCODE_EXECUTABLE ??
75+
(await downloadAndUnzipVSCode({ version: packageJson.devDependencies["@types/vscode"] }))
76+
const launch = (phase: "seed" | "measure") =>
77+
_electron.launch({
78+
executablePath,
79+
args: [
80+
workspace,
81+
`--extensionDevelopmentPath=${path.join(results, "builds", testInfo.project.name, "src")}`,
82+
`--extensionTestsPath=${path.join(packageRoot, "out/heap/sceneController")}`,
83+
`--user-data-dir=${userData}`,
84+
`--extensions-dir=${extensions}`,
85+
"--disable-workspace-trust",
86+
"--skip-welcome",
87+
"--skip-release-notes",
88+
"--disable-updates",
89+
"--disable-gpu",
90+
"--force-device-scale-factor=1",
91+
],
92+
env: {
93+
...process.env,
94+
AIMOCK_URL: mock.url,
95+
HEAP_RUN_DIR: temporaryRoot,
96+
HEAP_WORKLOAD: workload,
97+
HEAP_PHASE: phase,
98+
HTTP_PROXY: "http://127.0.0.1:9",
99+
HTTPS_PROXY: "http://127.0.0.1:9",
100+
ALL_PROXY: "http://127.0.0.1:9",
101+
NO_PROXY: "127.0.0.1,localhost",
102+
},
103+
locale: "en-US",
104+
timezoneId: "UTC",
105+
timeout: 60_000,
106+
})
107+
108+
app = await launch("seed")
109+
await waitForFile(path.join(temporaryRoot, "seed.ready"))
110+
await fs.writeFile(path.join(temporaryRoot, "seed.close"), "close")
111+
await app.close()
112+
app = undefined
113+
// No extension host is alive while replacing the persisted transcript.
114+
const seed = JSON.parse(await fs.readFile(path.join(temporaryRoot, "seed.ready"), "utf8")) as {
115+
largeTaskId: string
116+
}
117+
await fs.writeFile(
118+
path.join(temporaryRoot, "storage/tasks", seed.largeTaskId, "ui_messages.json"),
119+
JSON.stringify(createTranscript(workload)),
120+
)
121+
app = await launch("measure")
122+
const page = await app.firstWindow()
123+
await app.evaluate(({ BrowserWindow }) =>
124+
BrowserWindow.getAllWindows()[0]?.setBounds({ x: 0, y: 0, width: 1200, height: 800 }),
125+
)
126+
await waitForFile(path.join(temporaryRoot, "empty.ready"))
127+
let content: Frame | undefined
128+
await expect
129+
.poll(() => {
130+
const parent = page
131+
.frames()
132+
.find((frame) => frame.url().includes("extensionId=ZooCodeOrganization.zoo-code"))
133+
content = parent?.childFrames().find((frame) => frame.url().includes("/fake.html"))
134+
return Boolean(content)
135+
})
136+
.toBe(true)
137+
if (!content) throw new Error("Zoo Code webview content frame not found")
138+
// The content iframe may share its CDP target with the outer webview.
139+
let targetFrame: Frame | null = content
140+
while (targetFrame) {
141+
try {
142+
client = await app.context().newCDPSession(targetFrame)
143+
break
144+
} catch (error) {
145+
if (!(error instanceof Error) || !error.message.includes("does not have a separate CDP session"))
146+
throw error
147+
targetFrame = targetFrame.parentFrame()
148+
}
149+
}
150+
if (!client) throw new Error("No CDP target found for the webview")
151+
await client.send("HeapProfiler.enable")
152+
const target = await client.send("Target.getTargetInfo")
153+
expect(target.targetInfo.url).toContain("vscode-webview")
154+
const stages: Record<string, unknown> = {}
155+
for (const stage of ["empty", "loaded", "switched"]) {
156+
await waitForFile(path.join(temporaryRoot, `${stage}.ready`))
157+
if (stage !== "empty") {
158+
await expect(
159+
content
160+
.getByText(stage === "loaded" ? LARGE_TASK_MARKER : `${SMALL_TASK_MARKER}_COMPLETED`, {
161+
exact: false,
162+
})
163+
.first(),
164+
).toBeVisible()
165+
}
166+
await content.evaluate(async () => {
167+
await document.fonts.ready
168+
await new Promise<void>((resolve) =>
169+
requestAnimationFrame(() => requestAnimationFrame(() => resolve())),
170+
)
171+
})
172+
stages[stage] = await capture(client, testInfo.outputPath(`${stage}.heapsnapshot`))
173+
await fs.writeFile(path.join(temporaryRoot, `${stage}.continue`), "continue")
174+
}
175+
await waitForFile(path.join(temporaryRoot, "done"))
176+
const measurement = {
177+
variant: testInfo.project.name,
178+
workload,
179+
repeat: testInfo.repeatEachIndex,
180+
vscode: packageJson.devDependencies["@types/vscode"],
181+
target,
182+
stages,
183+
}
184+
await fs.writeFile(testInfo.outputPath("measurement.json"), JSON.stringify(measurement, null, 2))
185+
await testInfo.attach("measurement", { body: JSON.stringify(measurement), contentType: "application/json" })
186+
} catch (error) {
187+
if (app)
188+
await (
189+
await app.firstWindow()
190+
)
191+
.screenshot({ path: testInfo.outputPath("failure.png") })
192+
.catch(() => undefined)
193+
await fs
194+
.cp(path.join(userData, "logs"), testInfo.outputPath("logs"), { recursive: true })
195+
.catch(() => undefined)
196+
throw error
197+
} finally {
198+
await fs.writeFile(path.join(temporaryRoot, "close"), "close").catch(() => undefined)
199+
await client?.detach().catch(() => undefined)
200+
await app?.close().catch(() => undefined)
201+
await mock.stop().catch(() => undefined)
202+
await fs.rm(temporaryRoot, { recursive: true, force: true })
203+
}
204+
})
205+
}

0 commit comments

Comments
 (0)