Skip to content

Commit d35ebe0

Browse files
committed
test(app): carry the parallel-session benchmark for measurement
Temporary: it is anomalyco#54328. Used here to measure the markdown store cost in anomalyco#54305.
1 parent 65932e8 commit d35ebe0

1 file changed

Lines changed: 261 additions & 0 deletions

File tree

Lines changed: 261 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,261 @@
1+
import type { Page } from "@playwright/test"
2+
import type { OpenCodeEvent, SessionMessageInfo } from "@opencode/client/promise"
3+
import { benchmark, expect } from "../benchmark"
4+
import { mockOpenCodeServer } from "../../utils/mock-server"
5+
import { expectSessionTitle } from "../../utils/waits"
6+
import { fixture, installStressSessionTabs } from "../../utils/session-fixture"
7+
import { SERVER, sessionHref } from "../../utils/app"
8+
9+
// Parallel sessions on one server: every session streams deltas at the same time, so the
10+
// renderer pays the per-event cost of the whole fan-out, not of one session.
11+
const sessionCount = Number(process.env.PARALLEL_SESSIONS ?? 8)
12+
13+
const deltasPerSession = Number(process.env.PARALLEL_DELTAS ?? 200)
14+
15+
// Deltas for the visible session; 0 isolates the fan-out from the visible render.
16+
const activeDeltas = Number(process.env.PARALLEL_ACTIVE_DELTAS ?? deltasPerSession)
17+
18+
// Sessions spread over this many directories; each directory owns one child store.
19+
const directoryCount = Number(process.env.PARALLEL_DIRECTORIES ?? 1)
20+
21+
const completionMarker = "parallel-complete"
22+
23+
const directoryFor = (index: number) =>
24+
directoryCount <= 1 ? fixture.directory : `${fixture.directory}/ws-${index % directoryCount}`
25+
26+
const sessions = Array.from({ length: sessionCount }, (_, index) => ({
27+
...fixture.sessions[0],
28+
id: `ses_parallel_${index}`,
29+
slug: `parallel-${index}`,
30+
title: `Parallel session ${index}`,
31+
directory: directoryFor(index),
32+
}))
33+
34+
const assistantMessageID = (sessionID: string) => `msg_assistant_${sessionID}`
35+
36+
const textPartID = (sessionID: string) => `${assistantMessageID(sessionID)}:text:0`
37+
38+
function history(sessionID: string): SessionMessageInfo[] {
39+
return [
40+
{
41+
id: `msg_user_${sessionID}`,
42+
type: "user",
43+
time: { created: 1700000000000 },
44+
text: "Stream a long answer.",
45+
},
46+
{
47+
id: assistantMessageID(sessionID),
48+
type: "assistant",
49+
time: { created: 1700000001000 },
50+
model: { id: "claude-opus-4-6", providerID: "opencode", variant: "max" },
51+
agent: "build",
52+
cost: 0,
53+
tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } },
54+
finish: "stop",
55+
content: [{ type: "text", text: "Streaming" }],
56+
},
57+
]
58+
}
59+
60+
function deltaEvent(sessionID: string, directory: string, index: number, delta: string): OpenCodeEvent {
61+
return {
62+
id: `evt_parallel_${sessionID}_${index}`,
63+
created: 1700000002000 + index,
64+
type: "session.text.delta",
65+
data: { sessionID, assistantMessageID: assistantMessageID(sessionID), ordinal: 0, delta },
66+
location: { directory },
67+
} as OpenCodeEvent
68+
}
69+
70+
// Round-robin, so the burst interleaves sessions the way a busy server delivers them.
71+
// The payloads cross `page.evaluate`, so they stay opaque to the type checker.
72+
function buildBurst(): unknown[] {
73+
const events: unknown[] = []
74+
75+
for (let index = 0; index < deltasPerSession; index++) {
76+
for (const session of sessions) {
77+
if (session === sessions[0] && index >= activeDeltas) continue
78+
events.push(deltaEvent(session.id, session.directory, index, `chunk ${index} `))
79+
}
80+
}
81+
82+
// The last event lands in the active session, so its rendered text proves the whole
83+
// batch was applied before the probe stops the clock.
84+
events.push(deltaEvent(sessions[0]!.id, sessions[0]!.directory, deltasPerSession, ` ${completionMarker}`))
85+
86+
return events
87+
}
88+
89+
type ParallelProbeState = {
90+
started: number
91+
completed: number
92+
longTasks: number[]
93+
frames: number[]
94+
running: boolean
95+
cleanup: () => void
96+
start: (payloads: readonly unknown[]) => void
97+
}
98+
99+
type ParallelProbeWindow = Window & {
100+
__parallelSessionProbe?: ParallelProbeState
101+
__mockServerStreams?: Record<string, { push: (payloads: readonly unknown[]) => void }>
102+
}
103+
104+
benchmark.use({
105+
viewport: { width: 1440, height: 900 },
106+
video: "off",
107+
trace: "off",
108+
traceScope: "interaction",
109+
serviceWorkers: "block",
110+
})
111+
112+
const title = `parallel sessions: ${sessionCount} sessions x ${deltasPerSession} deltas over ${directoryCount} directories`
113+
114+
benchmark(title, async ({ page, report }) => {
115+
benchmark.setTimeout(120_000)
116+
const errors: string[] = []
117+
const prefetched = new Set<string>()
118+
const directories = new Set<string>()
119+
page.on("pageerror", (error) => errors.push(error.message))
120+
page.on("request", (request) => {
121+
const url = new URL(request.url())
122+
const match = url.pathname.match(/^\/api\/session\/([^/]+)\/form$/)
123+
124+
if (match) prefetched.add(match[1]!)
125+
const directory = url.searchParams.get("location[directory]")
126+
127+
if (directory) directories.add(directory)
128+
})
129+
await mockOpenCodeServer(page, {
130+
...fixture,
131+
sessions,
132+
pageMessages: (id) => ({ items: history(id) }),
133+
})
134+
await installStressSessionTabs(page, { sessionIDs: sessions.map((session) => session.id) })
135+
await page.goto(sessionHref(sessions[0]!.id))
136+
await expectSessionTitle(page, sessions[0]!.title)
137+
138+
// Visit every tab once so each session's transcript is loaded, then return to the first.
139+
for (const session of sessions.slice(1)) {
140+
await page.locator(`[data-slot="titlebar-tabs"] a[href="${sessionHref(session.id)}"]`).click()
141+
await expectSessionTitle(page, session.title)
142+
}
143+
144+
await page.locator(`[data-slot="titlebar-tabs"] a[href="${sessionHref(sessions[0]!.id)}"]`).click()
145+
await expectSessionTitle(page, sessions[0]!.title)
146+
// Inactive tabs prefetch attention, metadata, and the inbox on a timer; the burst must
147+
// not race that work.
148+
await expect.poll(() => sessions.slice(1).every((session) => prefetched.has(session.id))).toBe(true)
149+
await expect(page.locator(`[data-timeline-part-id="${textPartID(sessions[0]!.id)}"]`)).toBeVisible()
150+
151+
const cdp = await page.context().newCDPSession(page)
152+
await cdp.send("Performance.enable")
153+
const taskBefore = await taskDurationMs(cdp)
154+
await installParallelProbe(page, {
155+
server: SERVER,
156+
textPartID: textPartID(sessions[0]!.id),
157+
marker: completionMarker,
158+
})
159+
const events = buildBurst()
160+
await page.evaluate((payloads) => {
161+
;(window as ParallelProbeWindow).__parallelSessionProbe!.start(payloads)
162+
}, events)
163+
await page.waitForFunction(() => (window as ParallelProbeWindow).__parallelSessionProbe!.completed > 0)
164+
const taskAfter = await taskDurationMs(cdp)
165+
const metrics = await collectParallelProbe(page)
166+
167+
expect(metrics.elapsedMs).toBeGreaterThan(0)
168+
expect(metrics.frameCount).toBeGreaterThan(0)
169+
expect(errors).toEqual([])
170+
report(
171+
{ ...metrics, taskMs: taskAfter - taskBefore, events: events.length },
172+
{
173+
sessions: sessionCount,
174+
deltasPerSession,
175+
activeDeltas,
176+
events: events.length,
177+
directories: directories.size,
178+
directoryCount,
179+
transport: "playwright-route",
180+
browser: page.context().browser()!.version(),
181+
scope: "production app renderer; every session streaming",
182+
},
183+
)
184+
await cdp.detach()
185+
})
186+
187+
async function taskDurationMs(cdp: Awaited<ReturnType<Awaited<ReturnType<Page["context"]>>["newCDPSession"]>>) {
188+
const metrics = await cdp.send("Performance.getMetrics")
189+
190+
return metrics.metrics.find((metric) => metric.name === "TaskDuration")!.value * 1000
191+
}
192+
193+
async function installParallelProbe(page: Page, input: { server: string; textPartID: string; marker: string }) {
194+
await page.evaluate(({ server, textPartID, marker }) => {
195+
const host = window as ParallelProbeWindow
196+
const state: ParallelProbeState = {
197+
started: 0,
198+
completed: 0,
199+
longTasks: [],
200+
frames: [],
201+
running: false,
202+
cleanup: () => {},
203+
start: () => {},
204+
}
205+
const longTasks = new PerformanceObserver((list) => {
206+
for (const entry of list.getEntries()) state.longTasks.push(entry.duration)
207+
})
208+
longTasks.observe({ type: "longtask", buffered: false })
209+
// The streaming row can remount, so watch the document and re-query the part.
210+
const mutations = new MutationObserver(() => {
211+
if (state.completed) return
212+
const part = document.querySelector<HTMLElement>(`[data-timeline-part-id="${textPartID}"]`)
213+
214+
if (part?.textContent?.includes(marker)) state.completed = performance.now()
215+
})
216+
mutations.observe(document.body, { subtree: true, characterData: true, childList: true })
217+
let previous = 0
218+
const sample = (now: number) => {
219+
if (!state.running) return
220+
if (previous) state.frames.push(now - previous)
221+
previous = now
222+
requestAnimationFrame(sample)
223+
}
224+
state.start = (payloads) => {
225+
const stream = host.__mockServerStreams?.[server]
226+
227+
if (!stream) throw new Error(`missing mock event stream for ${server}`)
228+
state.started = performance.now()
229+
state.running = true
230+
requestAnimationFrame(sample)
231+
stream.push(payloads)
232+
}
233+
state.cleanup = () => {
234+
longTasks.disconnect()
235+
mutations.disconnect()
236+
state.running = false
237+
}
238+
host.__parallelSessionProbe = state
239+
}, input)
240+
}
241+
242+
async function collectParallelProbe(page: Page) {
243+
return page.evaluate(() => {
244+
const state = (window as ParallelProbeWindow).__parallelSessionProbe
245+
246+
if (!state) throw new Error("missing parallel session probe")
247+
state.cleanup()
248+
const sorted = state.frames.slice().sort((a, b) => a - b)
249+
const percentile = (value: number) => sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * value))] ?? null
250+
251+
return {
252+
elapsedMs: state.completed - state.started,
253+
longTaskMs: state.longTasks.reduce((sum, duration) => sum + duration, 0),
254+
longTaskCount: state.longTasks.length,
255+
frameCount: sorted.length,
256+
frameGapP50Ms: percentile(0.5),
257+
frameGapP95Ms: percentile(0.95),
258+
frameGapMaxMs: sorted.at(-1) ?? null,
259+
}
260+
})
261+
}

0 commit comments

Comments
 (0)