Skip to content

Commit 2ca9e57

Browse files
author
Zoo (VP)
committed
refactor(dashboard): address CodeRabbit review feedback
1 parent 38011c5 commit 2ca9e57

11 files changed

Lines changed: 149 additions & 182 deletions

‎src/services/stats/UsageEventStore.ts‎

Lines changed: 41 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,10 @@ const SEGMENT_MAX_BYTES = 5 * 1024 * 1024 // 5 MiB
1717
const TOTAL_MAX_BYTES = 100 * 1024 * 1024 // 100 MiB
1818

1919
/** Segment file name prefix */
20-
const SEGMENT_PREFIX = "events-"
20+
export const SEGMENT_PREFIX = "events-"
2121

2222
/** Segment file extension */
23-
const SEGMENT_EXT = ".ndjson"
23+
export const SEGMENT_EXT = ".ndjson"
2424

2525
/** Manifest file name */
2626
const MANIFEST_FILENAME = "manifest.json"
@@ -95,7 +95,7 @@ export interface QuarantineReportEntry {
9595
segment: string
9696
/** 1-based line number */
9797
line: number
98-
/** SHA-256 hash of the corrupt line content (first 16 chars) */
98+
/** 32-bit rolling hash of the corrupt line content as an 8-character hex string */
9999
hash: string
100100
/** Discovery time (ISO 8601 UTC) */
101101
at: string
@@ -247,13 +247,18 @@ export class UsageEventStore {
247247
// Incremental cache update: only after durable success.
248248
if (result && this.cachedEvents) {
249249
// If a segment rotation happened during the append, the cached
250-
// segment count no longer matches the manifest. Invalidate so the
250+
// segment count no longer matches the disk files. Invalidate so the
251251
// next readAll rescans from disk instead of returning stale data.
252+
const currentSegmentFiles = await this.listSegmentFiles()
252253
const manifest = await this.loadOrCreateManifest()
253-
if (this.cachedSegmentCount !== manifest.currentSegment) {
254+
if (this.cachedSegmentCount !== currentSegmentFiles.length) {
254255
this.invalidateCache()
255256
} else {
256257
this.cachedEvents.push(event)
258+
const activeSegmentPath = this.getSegmentPath(manifest.currentSegment)
259+
const activeStat = await fs.stat(activeSegmentPath).catch(() => null)
260+
this.cachedActiveSegmentSize = activeStat?.size ?? -1
261+
this.cachedActiveSegmentMtimeMs = activeStat?.mtimeMs ?? -1
257262
}
258263
}
259264
resolveFn(result)
@@ -299,7 +304,7 @@ export class UsageEventStore {
299304
this.cachedActiveSegmentSize === activeSize &&
300305
this.cachedActiveSegmentMtimeMs === activeMtimeMs
301306
) {
302-
return this.cachedEvents
307+
return [...this.cachedEvents]
303308
}
304309

305310
// Single-flight cold load: concurrent callers share one scan.
@@ -313,7 +318,7 @@ export class UsageEventStore {
313318
this.cachedSegmentCount = currentSegmentFiles.length
314319
this.cachedActiveSegmentSize = activeSize
315320
this.cachedActiveSegmentMtimeMs = activeMtimeMs
316-
return events
321+
return [...events]
317322
})
318323

319324
try {
@@ -344,7 +349,7 @@ export class UsageEventStore {
344349
}
345350

346351
try {
347-
const manifest = await this.loadOrCreateManifest()
352+
const manifest = await this.loadOrCreateManifest(true)
348353

349354
// New generation number
350355
const newGeneration = manifest.generation + 1
@@ -456,31 +461,26 @@ export class UsageEventStore {
456461

457462
for (const segmentFile of segmentFiles) {
458463
const segmentPath = path.join(this.statsDir, segmentFile)
459-
let content: string
460464

465+
let content: string
461466
try {
462467
content = await fs.readFile(segmentPath, "utf-8")
463468
} catch (err) {
464-
// Skip file read failures (ENOENT etc.)
465-
if ((err as NodeJS.ErrnoException).code !== "ENOENT") {
466-
console.warn(`[UsageEventStore] failed to read segment ${segmentFile}:`, err)
467-
}
468-
continue
469+
throw new StatsStoreError("STATS_STORE/readAll/002", `Failed to read segment file: ${segmentPath}`, err)
469470
}
470471

471472
const lines = content.split("\n")
472-
// Remove the last empty line (trailing newline)
473+
// Remove the last element if it's empty (trailing newline)
473474
if (lines.length > 0 && lines[lines.length - 1] === "") {
474475
lines.pop()
475476
}
476477

477-
// If the last line is unterminated/truncated, treat it as a crash tail and ignore
478-
// (If the last line is valid JSON, it is parsed; otherwise it goes to quarantine)
479478
for (let i = 0; i < lines.length; i++) {
480-
const lineNum = i + 1
481479
const line = lines[i]
480+
const lineNum = i + 1
482481
const isLastLine = i === lines.length - 1
483482

483+
// Skip empty lines
484484
if (!line.trim()) {
485485
continue
486486
}
@@ -491,11 +491,13 @@ export class UsageEventStore {
491491
if (result.success) {
492492
events.push(result.data)
493493
} else {
494-
// zod validation failed: corrupt line
495-
quarantineEntries.push(this.makeQuarantineEntry(segmentFile, lineNum, line))
496-
// Validation failure of the last line may be a crash tail, so exclude from quarantine
497-
if (isLastLine) {
498-
quarantineEntries.pop()
494+
// Schema validation failed
495+
// If it's the last line, it might be an unterminated/truncated crash tail.
496+
// The architecture report states: "The last unterminated line is treated
497+
// as a crash tail and ignored; parse failures before EOF are recorded to
498+
// the quarantine report and skipped."
499+
if (!isLastLine) {
500+
quarantineEntries.push(this.makeQuarantineEntry(segmentFile, lineNum, line))
499501
}
500502
}
501503
} catch {
@@ -560,7 +562,7 @@ export class UsageEventStore {
560562
}
561563

562564
try {
563-
const manifest = await this.loadOrCreateManifest()
565+
const manifest = await this.loadOrCreateManifest(true)
564566
const segmentPath = this.getSegmentPath(manifest.currentSegment)
565567

566568
// Check if the segment file exists and its size
@@ -628,16 +630,26 @@ export class UsageEventStore {
628630
// ── Internal: Manifest ──────────────────────────────────────────────────
629631

630632
private manifest: UsageStatsManifest | null = null
633+
private manifestMtimeMs = -1
631634

632635
/**
633636
* Loads the manifest or creates it with default values.
637+
* @param forceReload When true, bypasses the in-memory cached manifest and re-reads from disk.
634638
*/
635-
private async loadOrCreateManifest(): Promise<UsageStatsManifest> {
636-
if (this.manifest) {
637-
return this.manifest
639+
private async loadOrCreateManifest(forceReload = false): Promise<UsageStatsManifest> {
640+
if (!forceReload && this.manifest) {
641+
try {
642+
const stat = await fs.stat(this.manifestPath)
643+
if (stat.mtimeMs === this.manifestMtimeMs) {
644+
return this.manifest
645+
}
646+
} catch {
647+
// file stat failed, fall through to re-read or create
648+
}
638649
}
639650

640651
try {
652+
const stat = await fs.stat(this.manifestPath).catch(() => null)
641653
const content = await fs.readFile(this.manifestPath, "utf-8")
642654
const parsed = JSON.parse(content)
643655
// Basic field validation
@@ -647,6 +659,7 @@ export class UsageEventStore {
647659
typeof parsed.currentSegment === "number"
648660
) {
649661
this.manifest = parsed as UsageStatsManifest
662+
this.manifestMtimeMs = stat?.mtimeMs ?? -1
650663
return this.manifest
651664
}
652665
// On validation failure, overwrite with default values
@@ -664,6 +677,7 @@ export class UsageEventStore {
664677
console.warn(`[UsageEventStore] failed to load manifest, using default:`, err)
665678
const defaultManifest = { ...DEFAULT_MANIFEST, updatedAt: new Date().toISOString() }
666679
this.manifest = defaultManifest
680+
this.manifestMtimeMs = -1
667681
return defaultManifest
668682
}
669683
}

‎src/services/stats/UsageStatsMigration.ts‎

Lines changed: 14 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -5,18 +5,13 @@ import type { UsageEventV1 } from "@roo-code/types"
55
import { UsageEventV1 as UsageEventV1Schema } from "@roo-code/types"
66

77
import { UsageStatsDatabase, type MigrationCheckpoint } from "./UsageStatsDatabase"
8+
import { SEGMENT_PREFIX, SEGMENT_EXT } from "./UsageEventStore"
89

910
// ── Constants ──────────────────────────────────────────────────────────────
1011

1112
/** Number of events to migrate per batch. */
1213
const MIGRATION_BATCH_SIZE = 1000
1314

14-
/** Segment file name prefix (must match UsageEventStore). */
15-
const SEGMENT_PREFIX = "events-"
16-
17-
/** Segment file extension (must match UsageEventStore). */
18-
const SEGMENT_EXT = ".ndjson"
19-
2015
// ── Error Codes ─────────────────────────────────────────────────────────────
2116

2217
/**
@@ -76,7 +71,7 @@ export class UsageStatsMigration {
7671
* Runs the full migration from NDJSON segments to the SQLite database.
7772
* Resumes from the last checkpoint if interrupted.
7873
*
79-
* @returns The total number of events migrated (newly inserted).
74+
* @returns The total cumulative number of events migrated across checkpoints.
8075
*/
8176
migrate(): { totalMigrated: number; totalSkipped: number; complete: boolean } {
8277
const checkpoint = this.db.getMigrationCheckpoint()
@@ -99,12 +94,11 @@ export class UsageStatsMigration {
9994
}
10095

10196
// Build parent map for root task resolution
102-
// We need to read all events first to build the parent map,
103-
// but we do it in a streaming fashion: first pass collects
104-
// taskId → parentTaskId mappings, second pass migrates.
97+
// Reads segments to collect taskId → parentTaskId mappings for the migration.
10598
const parentMap = this.buildParentMap(segmentFiles)
10699

107-
let totalMigrated = checkpoint.eventsMigrated
100+
const initialMigrated = checkpoint.eventsMigrated
101+
let runMigrated = 0
108102
let totalSkipped = 0
109103

110104
// Determine starting point from checkpoint
@@ -198,14 +192,14 @@ export class UsageStatsMigration {
198192
// Flush batch when full
199193
if (batchCount >= MIGRATION_BATCH_SIZE) {
200194
const result = this.flushBatch(batchEvents)
201-
totalMigrated += result.migrated
195+
runMigrated += result.migrated
202196
totalSkipped += result.skipped
203197

204198
// Update checkpoint
205199
this.db.setMigrationCheckpoint({
206200
lastSegment: currentSegment,
207201
lastLine: currentLine,
208-
eventsMigrated: totalMigrated,
202+
eventsMigrated: initialMigrated + runMigrated,
209203
complete: false,
210204
})
211205

@@ -217,13 +211,13 @@ export class UsageStatsMigration {
217211
// Flush remaining events for this segment
218212
if (batchEvents.length > 0) {
219213
const result = this.flushBatch(batchEvents)
220-
totalMigrated += result.migrated
214+
runMigrated += result.migrated
221215
totalSkipped += result.skipped
222216

223217
this.db.setMigrationCheckpoint({
224218
lastSegment: currentSegment,
225219
lastLine: currentLine,
226-
eventsMigrated: totalMigrated,
220+
eventsMigrated: initialMigrated + runMigrated,
227221
complete: false,
228222
})
229223

@@ -239,11 +233,11 @@ export class UsageStatsMigration {
239233
this.db.setMigrationCheckpoint({
240234
lastSegment: currentSegment || segmentFiles[segmentFiles.length - 1],
241235
lastLine: currentLine,
242-
eventsMigrated: totalMigrated,
236+
eventsMigrated: initialMigrated + runMigrated,
243237
complete: true,
244238
})
245239

246-
return { totalMigrated, totalSkipped, complete: true }
240+
return { totalMigrated: initialMigrated + runMigrated, totalSkipped, complete: true }
247241
}
248242

249243
/**
@@ -307,9 +301,9 @@ export class UsageStatsMigration {
307301
// ── Internal: Parent Map ─────────────────────────────────────────────────
308302

309303
/**
310-
* Builds a map of taskId → parentTaskId from all segment files.
311-
* This is needed for root task resolution during migration.
312-
* Done in a streaming fashion to avoid loading all events into memory.
304+
* Builds a map of taskId → parentTaskId by reading all segment files.
305+
* Extracts only taskId and parentTaskId from each event to keep the map compact
306+
* for root task resolution during migration.
313307
*/
314308
private buildParentMap(segmentFiles: string[]): Map<string, string | undefined> {
315309
const parentMap = new Map<string, string | undefined>()

‎src/services/stats/__tests__/UsageEventStore.spec.ts‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,7 @@ import * as os from "os"
55
import { describe, it, expect, beforeEach, afterEach } from "vitest"
66

77
import type { UsageEventV1 } from "@roo-code/types"
8-
9-
import { UsageEventStore, StatsStoreError } from "../UsageEventStore"
8+
import { UsageEventStore } from "../UsageEventStore"
109

1110
// ── Test Helpers ────────────────────────────────────────────────────────────
1211

@@ -414,15 +413,14 @@ describe("UsageEventStore", () => {
414413
const events1 = await store.readAll()
415414
expect(events1).toHaveLength(1)
416415

417-
// Second readAll should return the same cached array reference
416+
// Second readAll should return the same cached array content
418417
const events2 = await store.readAll()
419-
expect(events2).toBe(events1)
418+
expect(events2).toEqual(events1)
420419
})
421420
})
422421

423422
describe("error handling", () => {
424-
it("should throw StatsStoreError with correct code on cap reached", async () => {
425-
// This test is difficult to force the cap, so only verify isCapped() method behavior
423+
it("should report not capped initially", async () => {
426424
expect(store.isCapped()).toBe(false)
427425
})
428426

‎src/services/stats/__tests__/UsageStatsDatabase.spec.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -334,10 +334,10 @@ describe("UsageStatsDatabase", () => {
334334
})
335335

336336
it("should update daily rollups", () => {
337-
const date = new Date(2026, 0, 15, 10, 0, 0)
338337
db.append(
339338
makeEvent({
340-
occurredAt: date.toISOString(),
339+
occurredAt: "2026-01-15T01:00:00.000Z",
340+
timezoneOffsetMinutes: 540,
341341
usage: {
342342
inputTokens: { value: 1000, source: "provider" },
343343
costUsd: { value: 0.05, source: "provider" },

‎src/services/stats/__tests__/UsageStatsService.spec.ts‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,12 @@ describe("UsageStatsService", () => {
7979
})
8080

8181
afterEach(async () => {
82+
try {
83+
await service?.dispose()
84+
} catch {
85+
// ignore disposal errors
86+
}
87+
vi.useRealTimers()
8288
// Clean up temp directory (test isolation)
8389
try {
8490
await fs.rm(tempDir, { recursive: true, force: true })
@@ -804,8 +810,6 @@ describe("UsageStatsService", () => {
804810
vi.advanceTimersByTime(6 * 60 * 1000)
805811

806812
await expect(service.clearStats(nonce)).rejects.toThrow(StatsServiceError)
807-
808-
vi.useRealTimers()
809813
})
810814

811815
it("should include error code STATS_SERVICE/clear/001 for expired nonce", async () => {
@@ -821,8 +825,6 @@ describe("UsageStatsService", () => {
821825
expect(err).toBeInstanceOf(StatsServiceError)
822826
expect((err as StatsServiceError).code).toBe("STATS_SERVICE/clear/001")
823827
}
824-
825-
vi.useRealTimers()
826828
})
827829

828830
it("should consume nonce after successful clear (one-time use)", async () => {
@@ -1242,8 +1244,6 @@ describe("UsageStatsService", () => {
12421244
vi.advanceTimersByTime(300)
12431245

12441246
expect(listener).toHaveBeenCalledOnce()
1245-
1246-
vi.useRealTimers()
12471247
})
12481248
})
12491249

0 commit comments

Comments
 (0)