|
| 1 | +import type { Page } from "@playwright/test" |
| 2 | +import type { OpenCodeEvent, SessionMessageInfo } from "@opencode/client/promise" |
| 3 | +import { benchmark, expect } from "../benchmark" |
| 4 | +import { mockOpenCodeServer } from "../../utils/mock-server" |
| 5 | +import { expectSessionTitle } from "../../utils/waits" |
| 6 | +import { fixture, installStressSessionTabs } from "../../utils/session-fixture" |
| 7 | +import { SERVER, sessionHref } from "../../utils/app" |
| 8 | + |
| 9 | +// Parallel sessions on one server: every session streams deltas at the same time, so the |
| 10 | +// renderer pays the per-event cost of the whole fan-out, not of one session. |
| 11 | +const sessionCount = Number(process.env.PARALLEL_SESSIONS ?? 8) |
| 12 | + |
| 13 | +const deltasPerSession = Number(process.env.PARALLEL_DELTAS ?? 200) |
| 14 | + |
| 15 | +// Deltas for the visible session; 0 isolates the fan-out from the visible render. |
| 16 | +const activeDeltas = Number(process.env.PARALLEL_ACTIVE_DELTAS ?? deltasPerSession) |
| 17 | + |
| 18 | +// Sessions spread over this many directories; each directory owns one child store. |
| 19 | +const directoryCount = Number(process.env.PARALLEL_DIRECTORIES ?? 1) |
| 20 | + |
| 21 | +const completionMarker = "parallel-complete" |
| 22 | + |
| 23 | +const directoryFor = (index: number) => |
| 24 | + directoryCount <= 1 ? fixture.directory : `${fixture.directory}/ws-${index % directoryCount}` |
| 25 | + |
| 26 | +const sessions = Array.from({ length: sessionCount }, (_, index) => ({ |
| 27 | + ...fixture.sessions[0], |
| 28 | + id: `ses_parallel_${index}`, |
| 29 | + slug: `parallel-${index}`, |
| 30 | + title: `Parallel session ${index}`, |
| 31 | + directory: directoryFor(index), |
| 32 | +})) |
| 33 | + |
| 34 | +const assistantMessageID = (sessionID: string) => `msg_assistant_${sessionID}` |
| 35 | + |
| 36 | +const textPartID = (sessionID: string) => `${assistantMessageID(sessionID)}:text:0` |
| 37 | + |
| 38 | +function history(sessionID: string): SessionMessageInfo[] { |
| 39 | + return [ |
| 40 | + { |
| 41 | + id: `msg_user_${sessionID}`, |
| 42 | + type: "user", |
| 43 | + time: { created: 1700000000000 }, |
| 44 | + text: "Stream a long answer.", |
| 45 | + }, |
| 46 | + { |
| 47 | + id: assistantMessageID(sessionID), |
| 48 | + type: "assistant", |
| 49 | + time: { created: 1700000001000 }, |
| 50 | + model: { id: "claude-opus-4-6", providerID: "opencode", variant: "max" }, |
| 51 | + agent: "build", |
| 52 | + cost: 0, |
| 53 | + tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } }, |
| 54 | + finish: "stop", |
| 55 | + content: [{ type: "text", text: "Streaming" }], |
| 56 | + }, |
| 57 | + ] |
| 58 | +} |
| 59 | + |
| 60 | +function deltaEvent(sessionID: string, directory: string, index: number, delta: string): OpenCodeEvent { |
| 61 | + return { |
| 62 | + id: `evt_parallel_${sessionID}_${index}`, |
| 63 | + created: 1700000002000 + index, |
| 64 | + type: "session.text.delta", |
| 65 | + data: { sessionID, assistantMessageID: assistantMessageID(sessionID), ordinal: 0, delta }, |
| 66 | + location: { directory }, |
| 67 | + } as OpenCodeEvent |
| 68 | +} |
| 69 | + |
| 70 | +// Round-robin, so the burst interleaves sessions the way a busy server delivers them. |
| 71 | +// The payloads cross `page.evaluate`, so they stay opaque to the type checker. |
| 72 | +function buildBurst(): unknown[] { |
| 73 | + const events: unknown[] = [] |
| 74 | + |
| 75 | + for (let index = 0; index < deltasPerSession; index++) { |
| 76 | + for (const session of sessions) { |
| 77 | + if (session === sessions[0] && index >= activeDeltas) continue |
| 78 | + events.push(deltaEvent(session.id, session.directory, index, `chunk ${index} `)) |
| 79 | + } |
| 80 | + } |
| 81 | + |
| 82 | + // The last event lands in the active session, so its rendered text proves the whole |
| 83 | + // batch was applied before the probe stops the clock. |
| 84 | + events.push(deltaEvent(sessions[0]!.id, sessions[0]!.directory, deltasPerSession, ` ${completionMarker}`)) |
| 85 | + |
| 86 | + return events |
| 87 | +} |
| 88 | + |
| 89 | +type ParallelProbeState = { |
| 90 | + started: number |
| 91 | + completed: number |
| 92 | + longTasks: number[] |
| 93 | + frames: number[] |
| 94 | + running: boolean |
| 95 | + cleanup: () => void |
| 96 | + start: (payloads: readonly unknown[]) => void |
| 97 | +} |
| 98 | + |
| 99 | +type ParallelProbeWindow = Window & { |
| 100 | + __parallelSessionProbe?: ParallelProbeState |
| 101 | + __mockServerStreams?: Record<string, { push: (payloads: readonly unknown[]) => void }> |
| 102 | +} |
| 103 | + |
| 104 | +benchmark.use({ |
| 105 | + viewport: { width: 1440, height: 900 }, |
| 106 | + video: "off", |
| 107 | + trace: "off", |
| 108 | + traceScope: "interaction", |
| 109 | + serviceWorkers: "block", |
| 110 | +}) |
| 111 | + |
| 112 | +const title = `parallel sessions: ${sessionCount} sessions x ${deltasPerSession} deltas over ${directoryCount} directories` |
| 113 | + |
| 114 | +benchmark(title, async ({ page, report }) => { |
| 115 | + benchmark.setTimeout(120_000) |
| 116 | + const errors: string[] = [] |
| 117 | + const prefetched = new Set<string>() |
| 118 | + const directories = new Set<string>() |
| 119 | + page.on("pageerror", (error) => errors.push(error.message)) |
| 120 | + page.on("request", (request) => { |
| 121 | + const url = new URL(request.url()) |
| 122 | + const match = url.pathname.match(/^\/api\/session\/([^/]+)\/form$/) |
| 123 | + |
| 124 | + if (match) prefetched.add(match[1]!) |
| 125 | + const directory = url.searchParams.get("location[directory]") |
| 126 | + |
| 127 | + if (directory) directories.add(directory) |
| 128 | + }) |
| 129 | + await mockOpenCodeServer(page, { |
| 130 | + ...fixture, |
| 131 | + sessions, |
| 132 | + pageMessages: (id) => ({ items: history(id) }), |
| 133 | + }) |
| 134 | + await installStressSessionTabs(page, { sessionIDs: sessions.map((session) => session.id) }) |
| 135 | + await page.goto(sessionHref(sessions[0]!.id)) |
| 136 | + await expectSessionTitle(page, sessions[0]!.title) |
| 137 | + |
| 138 | + // Visit every tab once so each session's transcript is loaded, then return to the first. |
| 139 | + for (const session of sessions.slice(1)) { |
| 140 | + await page.locator(`[data-slot="titlebar-tabs"] a[href="${sessionHref(session.id)}"]`).click() |
| 141 | + await expectSessionTitle(page, session.title) |
| 142 | + } |
| 143 | + |
| 144 | + await page.locator(`[data-slot="titlebar-tabs"] a[href="${sessionHref(sessions[0]!.id)}"]`).click() |
| 145 | + await expectSessionTitle(page, sessions[0]!.title) |
| 146 | + // Inactive tabs prefetch attention, metadata, and the inbox on a timer; the burst must |
| 147 | + // not race that work. |
| 148 | + await expect.poll(() => sessions.slice(1).every((session) => prefetched.has(session.id))).toBe(true) |
| 149 | + await expect(page.locator(`[data-timeline-part-id="${textPartID(sessions[0]!.id)}"]`)).toBeVisible() |
| 150 | + |
| 151 | + const cdp = await page.context().newCDPSession(page) |
| 152 | + await cdp.send("Performance.enable") |
| 153 | + const taskBefore = await taskDurationMs(cdp) |
| 154 | + await installParallelProbe(page, { |
| 155 | + server: SERVER, |
| 156 | + textPartID: textPartID(sessions[0]!.id), |
| 157 | + marker: completionMarker, |
| 158 | + }) |
| 159 | + const events = buildBurst() |
| 160 | + await page.evaluate((payloads) => { |
| 161 | + ;(window as ParallelProbeWindow).__parallelSessionProbe!.start(payloads) |
| 162 | + }, events) |
| 163 | + await page.waitForFunction(() => (window as ParallelProbeWindow).__parallelSessionProbe!.completed > 0) |
| 164 | + const taskAfter = await taskDurationMs(cdp) |
| 165 | + const metrics = await collectParallelProbe(page) |
| 166 | + |
| 167 | + expect(metrics.elapsedMs).toBeGreaterThan(0) |
| 168 | + expect(metrics.frameCount).toBeGreaterThan(0) |
| 169 | + expect(errors).toEqual([]) |
| 170 | + report( |
| 171 | + { ...metrics, taskMs: taskAfter - taskBefore, events: events.length }, |
| 172 | + { |
| 173 | + sessions: sessionCount, |
| 174 | + deltasPerSession, |
| 175 | + activeDeltas, |
| 176 | + events: events.length, |
| 177 | + directories: directories.size, |
| 178 | + directoryCount, |
| 179 | + transport: "playwright-route", |
| 180 | + browser: page.context().browser()!.version(), |
| 181 | + scope: "production app renderer; every session streaming", |
| 182 | + }, |
| 183 | + ) |
| 184 | + await cdp.detach() |
| 185 | +}) |
| 186 | + |
| 187 | +async function taskDurationMs(cdp: Awaited<ReturnType<Awaited<ReturnType<Page["context"]>>["newCDPSession"]>>) { |
| 188 | + const metrics = await cdp.send("Performance.getMetrics") |
| 189 | + |
| 190 | + return metrics.metrics.find((metric) => metric.name === "TaskDuration")!.value * 1000 |
| 191 | +} |
| 192 | + |
| 193 | +async function installParallelProbe(page: Page, input: { server: string; textPartID: string; marker: string }) { |
| 194 | + await page.evaluate(({ server, textPartID, marker }) => { |
| 195 | + const host = window as ParallelProbeWindow |
| 196 | + const state: ParallelProbeState = { |
| 197 | + started: 0, |
| 198 | + completed: 0, |
| 199 | + longTasks: [], |
| 200 | + frames: [], |
| 201 | + running: false, |
| 202 | + cleanup: () => {}, |
| 203 | + start: () => {}, |
| 204 | + } |
| 205 | + const longTasks = new PerformanceObserver((list) => { |
| 206 | + for (const entry of list.getEntries()) state.longTasks.push(entry.duration) |
| 207 | + }) |
| 208 | + longTasks.observe({ type: "longtask", buffered: false }) |
| 209 | + // The streaming row can remount, so watch the document and re-query the part. |
| 210 | + const mutations = new MutationObserver(() => { |
| 211 | + if (state.completed) return |
| 212 | + const part = document.querySelector<HTMLElement>(`[data-timeline-part-id="${textPartID}"]`) |
| 213 | + |
| 214 | + if (part?.textContent?.includes(marker)) state.completed = performance.now() |
| 215 | + }) |
| 216 | + mutations.observe(document.body, { subtree: true, characterData: true, childList: true }) |
| 217 | + let previous = 0 |
| 218 | + const sample = (now: number) => { |
| 219 | + if (!state.running) return |
| 220 | + if (previous) state.frames.push(now - previous) |
| 221 | + previous = now |
| 222 | + requestAnimationFrame(sample) |
| 223 | + } |
| 224 | + state.start = (payloads) => { |
| 225 | + const stream = host.__mockServerStreams?.[server] |
| 226 | + |
| 227 | + if (!stream) throw new Error(`missing mock event stream for ${server}`) |
| 228 | + state.started = performance.now() |
| 229 | + state.running = true |
| 230 | + requestAnimationFrame(sample) |
| 231 | + stream.push(payloads) |
| 232 | + } |
| 233 | + state.cleanup = () => { |
| 234 | + longTasks.disconnect() |
| 235 | + mutations.disconnect() |
| 236 | + state.running = false |
| 237 | + } |
| 238 | + host.__parallelSessionProbe = state |
| 239 | + }, input) |
| 240 | +} |
| 241 | + |
| 242 | +async function collectParallelProbe(page: Page) { |
| 243 | + return page.evaluate(() => { |
| 244 | + const state = (window as ParallelProbeWindow).__parallelSessionProbe |
| 245 | + |
| 246 | + if (!state) throw new Error("missing parallel session probe") |
| 247 | + state.cleanup() |
| 248 | + const sorted = state.frames.slice().sort((a, b) => a - b) |
| 249 | + const percentile = (value: number) => sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * value))] ?? null |
| 250 | + |
| 251 | + return { |
| 252 | + elapsedMs: state.completed - state.started, |
| 253 | + longTaskMs: state.longTasks.reduce((sum, duration) => sum + duration, 0), |
| 254 | + longTaskCount: state.longTasks.length, |
| 255 | + frameCount: sorted.length, |
| 256 | + frameGapP50Ms: percentile(0.5), |
| 257 | + frameGapP95Ms: percentile(0.95), |
| 258 | + frameGapMaxMs: sorted.at(-1) ?? null, |
| 259 | + } |
| 260 | + }) |
| 261 | +} |
0 commit comments