Skip to content

Commit 94d3a26

Browse files
author
Zoo (VP)
committed
test(e2e): add usage capture suite
1 parent 75fbbe5 commit 94d3a26

1 file changed

Lines changed: 219 additions & 0 deletions

File tree

Lines changed: 219 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,219 @@
1+
import * as assert from "assert"
2+
import * as fs from "fs/promises"
3+
import * as path from "path"
4+
5+
import * as vscode from "vscode"
6+
7+
import { UsageEventV1 } from "@roo-code/types"
8+
9+
import { setDefaultSuiteTimeout } from "./test-utils"
10+
import { waitFor, waitUntilCompleted } from "./utils"
11+
12+
/**
13+
* E2E coverage for the Token Usage Capture hooks (PR #1133).
14+
*
15+
* The extension wires a UsageRecorder into every Task. When an API attempt
16+
* reaches its terminal finalize boundary (completed / failed / cancelled),
17+
* the recorder appends a UsageEventV1 to the on-disk NDJSON store under
18+
* `<globalStorage>/usage-stats/events-*.ndjson`.
19+
*
20+
* These tests run a real task against the aimock-backed OpenRouter endpoint,
21+
* then read the segment files directly to assert that:
22+
* - a usage event was captured for the completed API call
23+
* - the capture hook recorded provider/model/token fields correctly
24+
* - idempotency keys are unique across events (no double-recording)
25+
*/
26+
27+
type UsageEvent = typeof UsageEventV1._output
28+
29+
const USAGE_STATS_DIRNAME = "usage-stats"
30+
const SEGMENT_PREFIX = "events-"
31+
const SEGMENT_EXT = ".ndjson"
32+
33+
/**
34+
* Read every usage event from the store's segment files.
35+
* Corrupt or unparseable lines are skipped — the store itself quarantines
36+
* them, so the test should not fail on them.
37+
*/
38+
const readAllUsageEvents = async (statsDir: string): Promise<UsageEvent[]> => {
39+
let files: string[]
40+
41+
try {
42+
files = await fs.readdir(statsDir)
43+
} catch {
44+
// Store directory does not exist yet — no events recorded.
45+
return []
46+
}
47+
48+
const segmentFiles = files.filter((f) => f.startsWith(SEGMENT_PREFIX) && f.endsWith(SEGMENT_EXT))
49+
const events: UsageEvent[] = []
50+
51+
for (const file of segmentFiles) {
52+
const content = await fs.readFile(path.join(statsDir, file), "utf-8")
53+
54+
for (const line of content.split("\n")) {
55+
const trimmed = line.trim()
56+
if (!trimmed) continue
57+
58+
try {
59+
const parsed = UsageEventV1.safeParse(JSON.parse(trimmed))
60+
if (parsed.success) {
61+
events.push(parsed.data)
62+
}
63+
} catch {
64+
// skip corrupt line
65+
}
66+
}
67+
}
68+
69+
return events
70+
}
71+
72+
suite("Roo Code Usage Capture", function () {
73+
setDefaultSuiteTimeout(this)
74+
75+
let statsDir: string
76+
77+
suiteSetup(async function () {
78+
// The extension writes usage events to
79+
// <userData>/User/globalStorage/ZooCodeOrganization.zoo-code/usage-stats/
80+
// (ExtensionContext.globalStorageUri.fsPath + "usage-stats").
81+
//
82+
// The test runner (@vscode/test-electron) launches VS Code with its
83+
// default --user-data-dir at <repoRoot>/.vscode-test/user-data unless
84+
// overridden. We resolve the directory by probing known candidates and
85+
// picking the first one that exists after the first task run; before
86+
// any event is written the directory may not exist yet, so the probe
87+
// falls back to the test-electron default.
88+
const extension = vscode.extensions.getExtension("ZooCodeOrganization.zoo-code")
89+
assert.ok(extension, "Extension not found")
90+
91+
const repoRoot = path.resolve(__dirname, "..", "..", "..")
92+
const candidates = [
93+
process.env.VSCODE_TEST_USER_DATA_DIR &&
94+
path.join(
95+
process.env.VSCODE_TEST_USER_DATA_DIR,
96+
"User",
97+
"globalStorage",
98+
"ZooCodeOrganization.zoo-code",
99+
USAGE_STATS_DIRNAME,
100+
),
101+
path.join(
102+
repoRoot,
103+
".vscode-test",
104+
"user-data",
105+
"User",
106+
"globalStorage",
107+
"ZooCodeOrganization.zoo-code",
108+
USAGE_STATS_DIRNAME,
109+
),
110+
].filter((c): c is string => !!c)
111+
112+
for (const candidate of candidates) {
113+
try {
114+
await fs.access(candidate)
115+
statsDir = candidate
116+
return
117+
} catch {
118+
// try next candidate
119+
}
120+
}
121+
122+
// Nothing written yet — use the test-electron default; the directory
123+
// will be created by UsageEventStore on first append.
124+
const fallback = candidates[candidates.length - 1]
125+
assert.ok(fallback, "At least one globalStorage candidate must be resolvable")
126+
statsDir = fallback
127+
})
128+
129+
test("captures a usage event when an API call completes", async () => {
130+
const api = globalThis.api
131+
132+
// Snapshot pre-existing event ids so we only assert on events this
133+
// test run created (the store persists across test runs).
134+
const preExistingIds = new Set((await readAllUsageEvents(statsDir)).map((e) => e.eventId))
135+
136+
const taskId = await waitUntilCompleted({
137+
api,
138+
start: () =>
139+
api.startNewTask({
140+
configuration: { mode: "ask", alwaysAllowModeSwitch: true, autoApprovalEnabled: true },
141+
text: "USAGE_CAPTURE_SMOKE: what is your name?",
142+
}),
143+
})
144+
145+
// finalizeUsageEvent is fire-and-forget from the task's perspective, so
146+
// poll the store until the event for this task appears.
147+
let eventsForTask: UsageEvent[] = []
148+
149+
await waitFor(async () => {
150+
const all = await readAllUsageEvents(statsDir)
151+
eventsForTask = all.filter((e) => !preExistingIds.has(e.eventId) && e.taskId === taskId)
152+
return eventsForTask.length > 0
153+
})
154+
155+
const completed = eventsForTask.find((e) => e.status === "completed")
156+
assert.ok(completed, `A completed usage event should be recorded for task ${taskId}`)
157+
158+
// The capture hook must record provider/model/mode provenance.
159+
assert.strictEqual(completed.provider, "openrouter", "Provider should match the configured apiProvider")
160+
assert.ok(completed.model.length > 0, "Model id should be recorded")
161+
assert.strictEqual(completed.mode, "ask", "Mode should match the task mode")
162+
163+
// Token usage captured from the aimock usage payload.
164+
assert.ok(
165+
(completed.usage.inputTokens?.value ?? 0) > 0,
166+
"Input tokens should be captured from the API usage payload",
167+
)
168+
assert.ok(
169+
(completed.usage.outputTokens?.value ?? 0) > 0,
170+
"Output tokens should be captured from the API usage payload",
171+
)
172+
assert.strictEqual(completed.usage.inputTokens?.source, "provider")
173+
assert.strictEqual(completed.usage.outputTokens?.source, "provider")
174+
175+
// Provenance + schema version invariants.
176+
assert.strictEqual(completed.schemaVersion, 1)
177+
assert.strictEqual(completed.provenance, "live")
178+
assert.ok(completed.idempotencyKey.startsWith(`${taskId}:`), "Idempotency key should embed the taskId")
179+
})
180+
181+
test("capture hook fires for each task and never double-records", async () => {
182+
const api = globalThis.api
183+
184+
const preExistingIds = new Set((await readAllUsageEvents(statsDir)).map((e) => e.eventId))
185+
186+
const taskId = await waitUntilCompleted({
187+
api,
188+
start: () =>
189+
api.startNewTask({
190+
configuration: { mode: "ask", alwaysAllowModeSwitch: true, autoApprovalEnabled: true },
191+
text: "USAGE_CAPTURE_HOOK_2: what is your name?",
192+
}),
193+
})
194+
195+
let eventsForTask: UsageEvent[] = []
196+
197+
await waitFor(async () => {
198+
const all = await readAllUsageEvents(statsDir)
199+
eventsForTask = all.filter((e) => !preExistingIds.has(e.eventId) && e.taskId === taskId)
200+
return eventsForTask.length > 0
201+
})
202+
203+
// Hook fired for this task too.
204+
assert.ok(
205+
eventsForTask.some((e) => e.status === "completed"),
206+
"Usage capture hook should fire for every completed task",
207+
)
208+
209+
// Idempotency: no two events anywhere in the store may share an
210+
// idempotencyKey — the recorder dedupes on requestKey:status.
211+
const all = await readAllUsageEvents(statsDir)
212+
const keys = all.map((e) => e.idempotencyKey)
213+
assert.strictEqual(
214+
new Set(keys).size,
215+
keys.length,
216+
"Idempotency keys must be unique (no double-recorded usage events)",
217+
)
218+
})
219+
})

0 commit comments

Comments
 (0)