Skip to content

Commit 0f33eab

Browse files
author
Zoo (VP)
committed
fix(stats): refine UsageEventStore idempotency, quarantine, hard cap, and clear cleanup
1 parent 2ca9e57 commit 0f33eab

2 files changed

Lines changed: 133 additions & 25 deletions

File tree

‎src/services/stats/UsageEventStore.ts‎

Lines changed: 48 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -384,6 +384,13 @@ export class UsageEventStore {
384384
// Save new manifest (safeWriteJson pattern: temp → rename)
385385
await this.writeManifestAtomic(newManifest)
386386

387+
// Clean up old-generation directory after successful manifest replacement
388+
try {
389+
await fs.rm(oldGenDir, { recursive: true, force: true })
390+
} catch (err) {
391+
console.warn(`[UsageEventStore] failed to clean up old generation dir ${oldGenDir}:`, err)
392+
}
393+
387394
// Reset idempotency set
388395
this.idempotencyKeys.clear()
389396
this.capped = false
@@ -469,6 +476,7 @@ export class UsageEventStore {
469476
throw new StatsStoreError("STATS_STORE/readAll/002", `Failed to read segment file: ${segmentPath}`, err)
470477
}
471478

479+
const endsWithNewline = content.endsWith("\n")
472480
const lines = content.split("\n")
473481
// Remove the last element if it's empty (trailing newline)
474482
if (lines.length > 0 && lines[lines.length - 1] === "") {
@@ -479,6 +487,7 @@ export class UsageEventStore {
479487
const line = lines[i]
480488
const lineNum = i + 1
481489
const isLastLine = i === lines.length - 1
490+
const isCrashTail = isLastLine && !endsWithNewline
482491

483492
// Skip empty lines
484493
if (!line.trim()) {
@@ -492,18 +501,17 @@ export class UsageEventStore {
492501
events.push(result.data)
493502
} else {
494503
// Schema validation failed
495-
// If it's the last line, it might be an unterminated/truncated crash tail.
496-
// The architecture report states: "The last unterminated line is treated
497-
// as a crash tail and ignored; parse failures before EOF are recorded to
498-
// the quarantine report and skipped."
499-
if (!isLastLine) {
504+
// If it's an unterminated/truncated crash tail at EOF, ignore.
505+
// Otherwise (including corrupted lines ending with newline), quarantine.
506+
if (!isCrashTail) {
500507
quarantineEntries.push(this.makeQuarantineEntry(segmentFile, lineNum, line))
501508
}
502509
}
503510
} catch {
504511
// JSON parse failed
505-
// Parse failure of the last line is treated as a crash tail and ignored
506-
if (!isLastLine) {
512+
// If it's an unterminated/truncated crash tail at EOF, ignore.
513+
// Otherwise (including corrupted lines ending with newline), quarantine.
514+
if (!isCrashTail) {
507515
quarantineEntries.push(this.makeQuarantineEntry(segmentFile, lineNum, line))
508516
}
509517
}
@@ -563,6 +571,26 @@ export class UsageEventStore {
563571

564572
try {
565573
const manifest = await this.loadOrCreateManifest(true)
574+
575+
// Idempotency double-check under lock (rebuild idempotency set from disk)
576+
await this.rebuildIdempotencySet(manifest)
577+
if (this.idempotencyKeys.has(event.idempotencyKey)) {
578+
return false
579+
}
580+
581+
// Prospective hard cap check under lock
582+
const line = JSON.stringify(event) + "\n"
583+
const serializedBytes = Buffer.byteLength(line, "utf-8")
584+
const currentTotalBytes = await this.getTotalSizeBytes()
585+
586+
if (currentTotalBytes + serializedBytes >= TOTAL_MAX_BYTES) {
587+
this.capped = true
588+
throw new StatsStoreError(
589+
"STATS_STORE/append/003",
590+
"Storage hard cap (100 MiB) reached, new events suspended",
591+
)
592+
}
593+
566594
const segmentPath = this.getSegmentPath(manifest.currentSegment)
567595

568596
// Check if the segment file exists and its size
@@ -590,9 +618,6 @@ export class UsageEventStore {
590618
// allowing a single segment to grow indefinitely.
591619
const activeSegmentPath = this.getSegmentPath(manifest.currentSegment)
592620

593-
// Append event as compact JSON + \n
594-
const line = JSON.stringify(event) + "\n"
595-
596621
try {
597622
// Open in append mode and write
598623
const handle = await fs.open(activeSegmentPath, "a")
@@ -615,7 +640,7 @@ export class UsageEventStore {
615640
this.idempotencyKeys.add(event.idempotencyKey)
616641

617642
// Check total size and update cap
618-
this.capped = await this.checkTotalSize()
643+
this.capped = currentTotalBytes + serializedBytes >= TOTAL_MAX_BYTES
619644

620645
return true
621646
} finally {
@@ -777,9 +802,9 @@ export class UsageEventStore {
777802
// ── Internal: Size Management ────────────────────────────────────────────
778803

779804
/**
780-
* Checks the total event file size and returns whether the hard cap has been reached.
805+
* Returns the total size in bytes of all segment files on disk.
781806
*/
782-
private async checkTotalSize(): Promise<boolean> {
807+
private async getTotalSizeBytes(): Promise<number> {
783808
try {
784809
const allFiles = await fs.readdir(this.statsDir)
785810
const segmentFiles = allFiles.filter((f) => f.startsWith(SEGMENT_PREFIX) && f.endsWith(SEGMENT_EXT))
@@ -794,12 +819,20 @@ export class UsageEventStore {
794819
}
795820
}
796821

797-
return totalSize >= TOTAL_MAX_BYTES
822+
return totalSize
798823
} catch {
799-
return false
824+
return 0
800825
}
801826
}
802827

828+
/**
829+
* Checks the total event file size and returns whether the hard cap has been reached.
830+
*/
831+
private async checkTotalSize(): Promise<boolean> {
832+
const totalBytes = await this.getTotalSizeBytes()
833+
return totalBytes >= TOTAL_MAX_BYTES
834+
}
835+
803836
// ── Internal: Quarantine ────────────────────────────────────────────────
804837

805838
/**

‎src/services/stats/__tests__/UsageEventStore.spec.ts‎

Lines changed: 85 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -235,31 +235,72 @@ describe("UsageEventStore", () => {
235235
expect(events[1].eventId).toBe("evt-2")
236236
})
237237

238-
it("should skip corrupt lines and continue reading", async () => {
238+
it("should quarantine corrupt last line if it ends with newline", async () => {
239239
const event = makeEvent()
240240
await store.append(event)
241241

242-
// Manually add a corrupt line
242+
// Add a corrupt line that ends with \n (fully written, not a crash tail)
243243
const segmentPath = path.join(store._getStatsDir(), "events-000001.ndjson")
244244
await fs.appendFile(segmentPath, "{invalid json line\n")
245245

246246
const events = await store.readAll()
247247
expect(events).toHaveLength(1) // corrupt line is skipped
248+
249+
const quarantinePath = path.join(store._getStatsDir(), "quarantine", "corrupt-lines.jsonl")
250+
const quarantineExists = await fs
251+
.access(quarantinePath)
252+
.then(() => true)
253+
.catch(() => false)
254+
expect(quarantineExists).toBe(true)
255+
256+
const quarantineContent = await fs.readFile(quarantinePath, "utf-8")
257+
const quarantineEntries = quarantineContent
258+
.trim()
259+
.split("\n")
260+
.map((l) => JSON.parse(l))
261+
expect(quarantineEntries).toHaveLength(1)
262+
expect(quarantineEntries[0].line).toBe(2)
263+
})
264+
265+
it("should quarantine schema-invalid last line if it ends with newline", async () => {
266+
const event = makeEvent()
267+
await store.append(event)
268+
269+
// Add a schema-invalid line ending with \n
270+
const segmentPath = path.join(store._getStatsDir(), "events-000001.ndjson")
271+
await fs.appendFile(segmentPath, JSON.stringify({ schemaVersion: 999, invalid: true }) + "\n")
272+
273+
const events = await store.readAll()
274+
expect(events).toHaveLength(1)
275+
276+
const quarantinePath = path.join(store._getStatsDir(), "quarantine", "corrupt-lines.jsonl")
277+
const quarantineExists = await fs
278+
.access(quarantinePath)
279+
.then(() => true)
280+
.catch(() => false)
281+
expect(quarantineExists).toBe(true)
248282
})
249283

250-
it("should ignore truncated last line (crash tail)", async () => {
284+
it("should ignore truncated last line (crash tail) without writing quarantine report", async () => {
251285
const event = makeEvent()
252286
await store.append(event)
253287

254-
// Manually add a truncated line (last line)
288+
// Manually add a truncated line (last line, no trailing newline)
255289
const segmentPath = path.join(store._getStatsDir(), "events-000001.ndjson")
256-
await fs.appendFile(segmentPath, '{"partial": tru') // Truncated JSON
290+
await fs.appendFile(segmentPath, '{"partial": tru') // Truncated JSON without \n
257291

258292
const events = await store.readAll()
259293
expect(events).toHaveLength(1) // crash tail is ignored
294+
295+
const quarantinePath = path.join(store._getStatsDir(), "quarantine", "corrupt-lines.jsonl")
296+
const quarantineExists = await fs
297+
.access(quarantinePath)
298+
.then(() => true)
299+
.catch(() => false)
300+
expect(quarantineExists).toBe(false)
260301
})
261302

262-
it("should write quarantine report for corrupt lines", async () => {
303+
it("should write quarantine report for corrupt lines in the middle", async () => {
263304
const event = makeEvent()
264305
await store.append(event)
265306

@@ -352,7 +393,7 @@ describe("UsageEventStore", () => {
352393
expect(result).toBe(true)
353394
})
354395

355-
it("should move old segments to old-generation directory", async () => {
396+
it("should clean up old-generation directory after clear", async () => {
356397
await store.append(makeEvent())
357398

358399
await store.clear()
@@ -362,11 +403,11 @@ describe("UsageEventStore", () => {
362403
.access(oldGenDir)
363404
.then(() => true)
364405
.catch(() => false)
365-
expect(oldGenExists).toBe(true)
406+
expect(oldGenExists).toBe(false)
366407
})
367408
})
368409

369-
describe("idempotency recovery on restart", () => {
410+
describe("idempotency recovery on restart and multi-instance", () => {
370411
it("should rebuild idempotency set from segment scan on re-init", async () => {
371412
const event = makeEvent({ idempotencyKey: "idem-persist" })
372413
await store.append(event)
@@ -379,6 +420,24 @@ describe("UsageEventStore", () => {
379420
const result = await newStore.append(event)
380421
expect(result).toBe(false)
381422
})
423+
424+
it("should double-check idempotency across concurrent store instances sharing same directory", async () => {
425+
const store2 = new UsageEventStore(tempDir)
426+
await store2.initialize()
427+
428+
const event = makeEvent({ idempotencyKey: "idem-cross-instance" })
429+
430+
// First append on store1 succeeds
431+
const result1 = await store.append(event)
432+
expect(result1).toBe(true)
433+
434+
// Second append on store2 with same idempotencyKey is rejected under lock
435+
const result2 = await store2.append(event)
436+
expect(result2).toBe(false)
437+
438+
const events = await store2.readAll()
439+
expect(events).toHaveLength(1)
440+
})
382441
})
383442

384443
describe("cache invalidation on same-segment append", () => {
@@ -419,7 +478,7 @@ describe("UsageEventStore", () => {
419478
})
420479
})
421480

422-
describe("error handling", () => {
481+
describe("error handling and hard cap", () => {
423482
it("should report not capped initially", async () => {
424483
expect(store.isCapped()).toBe(false)
425484
})
@@ -431,5 +490,21 @@ describe("UsageEventStore", () => {
431490
// Re-appending the same event is not an error
432491
await expect(store.append(event)).resolves.toBe(false)
433492
})
493+
494+
it("should prospectively reject appends when hard cap would be reached", async () => {
495+
const TOTAL_MAX_BYTES = 100 * 1024 * 1024 // 100 MiB
496+
const statsDir = store._getStatsDir()
497+
498+
// Pre-fill a dummy segment file that puts total size just 50 bytes below 100 MiB
499+
const dummySegment = path.join(statsDir, "events-000001.ndjson")
500+
const dummySize = TOTAL_MAX_BYTES - 50
501+
const buffer = Buffer.alloc(dummySize, " ")
502+
await fs.writeFile(dummySegment, buffer)
503+
504+
const event = makeEvent()
505+
// Appending an event (which serializes to > 100 bytes) must prospectively fail
506+
await expect(store.append(event)).rejects.toThrow("Storage hard cap (100 MiB) reached")
507+
expect(store.isCapped()).toBe(true)
508+
})
434509
})
435510
})

0 commit comments

Comments
 (0)