Skip to content

Commit 4c8b6cf

Browse files
committed
feat(webview): add explicit interface recovery retry with generation guards
1 parent c8335eb commit 4c8b6cf

29 files changed

Lines changed: 684 additions & 114 deletions

‎apps/vscode-e2e/src/visual/heartbeat.electron.ts‎

Lines changed: 97 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -121,77 +121,105 @@ test("recovers when heartbeat delivery stops and resumes periodic heartbeats", a
121121
}
122122
})
123123

124-
test("recovers after the isolated Zoo Code renderer is terminated", async ({ browserName: _browserName }, testInfo) => {
125-
// Allow startup, stale detection, recovery observations, and cleanup.
126-
test.setTimeout(8 * 60_000)
127-
const running = await startScene(
128-
vscodeExecutablePath,
129-
{ ...scenario, scene: "chat", startMock: true, landmark: CHAT_RESULT },
130-
testInfo,
131-
)
132-
try {
133-
expect(running.taskIds).toHaveLength(1)
134-
const initial = await waitForContent(running.page, CHAT_RESULT)
135-
await initial.evaluate(() => {
136-
document.documentElement.dataset.heartbeatDocument = "before-crash"
137-
})
138-
const processes = await running.app.evaluate(({ BrowserWindow }) => {
139-
const mainFrame = BrowserWindow.getAllWindows()[0]?.webContents.mainFrame
140-
if (!mainFrame) throw new Error("VS Code workbench frame not found")
141-
const renderer = mainFrame.framesInSubtree.find(
142-
(frame: { url: string; parent?: { url: string }; osProcessId: number }) =>
143-
frame.url.includes("/fake.html") &&
144-
frame.parent?.url.includes("extensionId=ZooCodeOrganization.zoo-code"),
124+
for (const explicitRetry of [false, true])
125+
test(
126+
explicitRetry
127+
? "explicit retry recovers the isolated Zoo Code renderer and retains its task"
128+
: "recovers after the isolated Zoo Code renderer is terminated",
129+
async ({ browserName: _browserName }, testInfo) => {
130+
// Allow startup, stale detection, recovery observations, and cleanup.
131+
test.setTimeout(8 * 60_000)
132+
const running = await startScene(
133+
vscodeExecutablePath,
134+
{ ...scenario, scene: "chat", startMock: true, landmark: CHAT_RESULT },
135+
testInfo,
145136
)
146-
if (!renderer) throw new Error("Zoo Code renderer process not found")
147-
return { workbench: mainFrame.osProcessId, webview: renderer.osProcessId }
148-
})
149-
await testInfo.attach("renderer-processes.json", {
150-
body: JSON.stringify(processes),
151-
contentType: "application/json",
152-
})
153-
// Never terminate the workbench: this test requires a separate OOPIF process.
154-
expect(processes.webview).toBeGreaterThan(0)
155-
expect(processes.webview).not.toBe(processes.workbench)
156-
process.kill(processes.webview, "SIGKILL")
137+
try {
138+
expect(running.taskIds).toHaveLength(1)
139+
const initial = await waitForContent(running.page, CHAT_RESULT)
140+
await initial.evaluate(() => {
141+
document.documentElement.dataset.heartbeatDocument = "before-crash"
142+
})
143+
if (explicitRetry) {
144+
// Establish a fresh delivered heartbeat before the crash. Explicit
145+
// recovery must finish before the 90-second stale threshold.
146+
const beforeCrash = await observeHeartbeats(running.page)
147+
await expect
148+
.poll(() => beforeCrash.evaluate((state) => state.forwarded), { timeout: 45_000 })
149+
.toBeGreaterThan(0)
150+
}
151+
const processes = await running.app.evaluate(({ BrowserWindow }) => {
152+
const mainFrame = BrowserWindow.getAllWindows()[0]?.webContents.mainFrame
153+
if (!mainFrame) throw new Error("VS Code workbench frame not found")
154+
const renderer = mainFrame.framesInSubtree.find(
155+
(frame: { url: string; parent?: { url: string }; osProcessId: number }) =>
156+
frame.url.includes("/fake.html") &&
157+
frame.parent?.url.includes("extensionId=ZooCodeOrganization.zoo-code"),
158+
)
159+
if (!renderer) throw new Error("Zoo Code renderer process not found")
160+
return { workbench: mainFrame.osProcessId, webview: renderer.osProcessId }
161+
})
162+
await testInfo.attach("renderer-processes.json", {
163+
body: JSON.stringify(processes),
164+
contentType: "application/json",
165+
})
166+
// Never terminate the workbench: this test requires a separate OOPIF process.
167+
expect(processes.webview).toBeGreaterThan(0)
168+
expect(processes.webview).not.toBe(processes.workbench)
169+
process.kill(processes.webview, "SIGKILL")
170+
if (explicitRetry) {
171+
await running.page.keyboard.press("F1")
172+
const input = running.page.locator(".quick-input-widget input")
173+
await expect(input).toBeVisible({ timeout: 10_000 })
174+
await input.fill(">Zoo Code: Retry Interface Recovery", { timeout: 10_000 })
175+
await running.page
176+
.locator(".quick-input-list .monaco-list-row")
177+
.filter({ hasText: "Retry Interface Recovery" })
178+
.first()
179+
.click({ timeout: 10_000 })
180+
}
157181

158-
// Allow stale detection (up to 150s), confirmation and replacement waits,
159-
// plus webview loading. A passing fast run does not bound the slow path.
160-
await expect
161-
.poll(
162-
async () => {
163-
try {
164-
const frame = contentFrame(running.page)
165-
if (!frame || !(await frame.getByText(CHAT_RESULT, { exact: false }).isVisible()))
166-
return "missing"
167-
return (
168-
(await frame.locator("html").getAttribute("data-heartbeat-document", { timeout: 1_000 })) ??
169-
"recovered"
170-
)
171-
} catch {
172-
return "unavailable"
173-
}
174-
},
175-
{ timeout: 210_000, intervals: [1_000] },
176-
)
177-
.toBe("recovered")
178-
await waitForContent(running.page, CHAT_RESULT)
179-
await fs.writeFile(`${running.donePath}.probe`, "probe")
180-
await expect
181-
.poll(async () => {
182-
const result = await fs.readFile(`${running.donePath}.probe.result`, "utf8").catch(() => "")
183-
return result ? (JSON.parse(result) as { taskIds: string[] }) : undefined
184-
})
185-
.toMatchObject({ taskIds: running.taskIds })
186-
const heartbeat = await observeHeartbeats(running.page)
187-
await expect.poll(() => heartbeat.evaluate((state) => state.forwarded), { timeout: 45_000 }).toBeGreaterThan(0)
188-
} catch (error) {
189-
await attachFailureArtifacts(running, testInfo)
190-
throw error
191-
} finally {
192-
await stopScene(running, testInfo)
193-
}
194-
})
182+
// Allow stale detection (up to 150s), confirmation and replacement waits,
183+
// plus webview loading. A passing fast run does not bound the slow path.
184+
await expect
185+
.poll(
186+
async () => {
187+
try {
188+
const frame = contentFrame(running.page)
189+
if (!frame || !(await frame.getByText(CHAT_RESULT, { exact: false }).isVisible()))
190+
return "missing"
191+
return (
192+
(await frame
193+
.locator("html")
194+
.getAttribute("data-heartbeat-document", { timeout: 1_000 })) ?? "recovered"
195+
)
196+
} catch {
197+
return "unavailable"
198+
}
199+
},
200+
{ timeout: explicitRetry ? 45_000 : 210_000, intervals: [1_000] },
201+
)
202+
.toBe("recovered")
203+
await waitForContent(running.page, CHAT_RESULT)
204+
await fs.writeFile(`${running.donePath}.probe`, "probe")
205+
await expect
206+
.poll(async () => {
207+
const result = await fs.readFile(`${running.donePath}.probe.result`, "utf8").catch(() => "")
208+
return result ? (JSON.parse(result) as { taskIds: string[] }) : undefined
209+
})
210+
.toMatchObject({ taskIds: running.taskIds })
211+
const heartbeat = await observeHeartbeats(running.page)
212+
await expect
213+
.poll(() => heartbeat.evaluate((state) => state.forwarded), { timeout: 45_000 })
214+
.toBeGreaterThan(0)
215+
} catch (error) {
216+
await attachFailureArtifacts(running, testInfo)
217+
throw error
218+
} finally {
219+
await stopScene(running, testInfo)
220+
}
221+
},
222+
)
195223

196224
test("keeps the same document when the sidebar is hidden past the stale threshold", async ({
197225
browserName: _browserName,

‎apps/vscode-e2e/src/visual/heartbeat.md‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,14 @@ The message-port interceptor follows the pinned VS Code implementation. Check it
1717

1818
A gray sidebar can persist after renderer termination even when the watchdog assigns fresh HTML. HTML delivery alone cannot restart a dead renderer. Recovery now waits ten seconds for a heartbeat after assigning HTML, then recreates the affected host if the renderer has not responded. Sidebar recovery uses the generated view-removal and focus commands after checking their availability; tab recovery replaces its panel while retaining its provider. The crash test covers the real sidebar path. Unit tests cover tab replacement and task ownership.
1919

20-
A recovery attempt includes HTML-generation errors. Three attempts exhaust the budget; a startup heartbeat alone does not reset it. Periodic heartbeats must span a 60-second healthy window to replenish the budget. Exhaustion or unavailable/failed scoped host recovery logs once and offers a non-modal Reload Window or Show Logs action. A sidebar replacement that arrives after the ten-second wait retains the incident budget and displayed task. A failed sidebar focus command clears the pending marker so a later manual open starts normally. A sidebar or tab that fails initialization after publication shows the recovery warning and keeps its task and a watchdog for bounded retries. Failure handling lives in the shared view resolver, including sidebar callbacks invoked independently by VS Code. A published sidebar that is still initializing after the ten-second wait keeps its remaining attempts. The warning does not consume the remaining attempts. Duplicate warnings are suppressed until heartbeats resume or a user-opened view resets the notification state. A later terminal failure still logs its final stop reason once, independently of warning deduplication. Subsequent watchdog ticks do not repeat the stop log. Only sustained healthy heartbeats replenish the incident budget. Window reload is never automatic. A late Reload Window action from an old notification is ignored after liveness returns. Show Logs still opens the output channel.
20+
A recovery attempt includes HTML-generation errors. Three attempts exhaust the budget; a startup heartbeat alone does not reset it. Periodic heartbeats must span a 60-second healthy window to replenish the budget. Exhaustion or unavailable/failed scoped host recovery logs once and offers non-modal Retry Interface Recovery, Reload Window, and Show Logs actions. A sidebar replacement that arrives after the ten-second wait retains the incident budget and displayed task. A failed sidebar focus command clears the pending marker so a later manual open starts normally. A sidebar or tab that fails initialization after publication shows the recovery warning and keeps its task and a watchdog for bounded retries. Failure handling lives in the shared view resolver, including sidebar callbacks invoked independently by VS Code. A published sidebar that is still initializing after the ten-second wait keeps its remaining attempts. The warning does not consume the remaining attempts. Duplicate warnings are suppressed until heartbeats resume or a user-opened view resets the notification state. A later terminal failure still logs its final stop reason once, independently of warning deduplication. Subsequent watchdog ticks do not repeat the stop log. Automatic recovery replenishes the incident budget only after sustained healthy heartbeats. Window reload is never automatic. Late Retry Interface Recovery and Reload Window actions are ignored after liveness returns or a newer explicit recovery generation starts. Show Logs still opens the output channel.
2121

2222
Keep exact timing boundaries, individual post-await guards, and reload-budget assertions in provider unit tests. Exercise OS minimize and sleep/resume behavior manually; the Electron scenarios do not reproduce OS suspension or its scheduling order.
2323

2424
The [bounded recovery model](../../../../docs/architecture/webview-recovery-model.md) checks production recovery policies and abstract interleavings. Run `pnpm webview-recovery:model-check` alongside the provider and Electron tests. It does not prove VS Code effects or recovery liveness.
25+
26+
## Explicit recovery after a timeout
27+
28+
Choose **Retry Interface Recovery** in the warning, or run **Zoo Code: Retry Interface Recovery** from the Command Palette. The command remains available if the warning was dismissed or the sidebar has no live view. It starts a new recovery generation, counts the requested repair as the first of three attempts, clears the old warning/log state, and retains the task stack, including an interrupted task. It does not reload the VS Code window.
29+
30+
Reopening a sidebar after a timed-out focus is ambiguous: VS Code does not pass a user-versus-programmatic origin to the resolver. Such a callback keeps the pending incident and its budget. The explicit retry command supplies the missing user intent. Late command completions and notifications from older generations cannot change the renewed budget or release its operation ownership. An already-issued VS Code command cannot be cancelled by this generation guard; the guard prevents subsequent obsolete extension-side effects.

0 commit comments

Comments
 (0)