Skip to content

Commit 417e143

Browse files
ryoppippiryoppippi
committed
refactor: use data-aggregation helpers in data-loader
Replace duplicate code in data-loader.ts with the new helper functions from data-aggregation module: - Use aggregateByModel() instead of inline model aggregation logic - Use createModelBreakdowns() for consistent breakdown formatting - Use calculateTotals() instead of inline reduce operations - Use filterByDateRange() for date filtering - Use isDuplicateEntry() and markAsProcessed() for deduplication - Use extractUniqueModels() for model list extraction - Use aggregateModelBreakdowns() in loadMonthlyUsageData This reduces code duplication by ~195 lines while maintaining the same functionality and improving maintainability.
1 parent 3451141 commit 417e143

File tree

1 file changed

+49
-187
lines changed

1 file changed

+49
-187
lines changed

‎src/data-loader.ts‎

Lines changed: 49 additions & 187 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,16 @@ import { unreachable } from '@core/errorutil';
66
import { sort } from 'fast-sort';
77
import { glob } from 'tinyglobby';
88
import * as v from 'valibot';
9+
import {
10+
aggregateByModel,
11+
aggregateModelBreakdowns,
12+
calculateTotals,
13+
createModelBreakdowns,
14+
extractUniqueModels,
15+
filterByDateRange,
16+
isDuplicateEntry,
17+
markAsProcessed,
18+
} from './data-aggregation.ts';
919
import { logger } from './logger.ts';
1020
import {
1121
PricingFetcher,
@@ -305,15 +315,13 @@ export async function loadDailyUsageData(
305315

306316
// Check for duplicate message + request ID combination
307317
const uniqueHash = createUniqueHash(data);
308-
if (uniqueHash != null && processedHashes.has(uniqueHash)) {
318+
if (isDuplicateEntry(uniqueHash, processedHashes)) {
309319
// Skip duplicate message
310320
continue;
311321
}
312322

313323
// Mark this combination as processed
314-
if (uniqueHash != null) {
315-
processedHashes.add(uniqueHash);
316-
}
324+
markAsProcessed(uniqueHash, processedHashes);
317325

318326
const date = formatDate(data.timestamp);
319327
// If fetcher is available, calculate cost based on mode and tokens
@@ -341,70 +349,24 @@ export async function loadDailyUsageData(
341349
}
342350

343351
// Aggregate by model first
344-
const modelAggregates = new Map<string, {
345-
inputTokens: number;
346-
outputTokens: number;
347-
cacheCreationTokens: number;
348-
cacheReadTokens: number;
349-
cost: number;
350-
}>();
351-
352-
for (const entry of entries) {
353-
const modelName = entry.model ?? 'unknown';
354-
// Skip synthetic model
355-
if (modelName === '<synthetic>') {
356-
continue;
357-
}
358-
const existing = modelAggregates.get(modelName) ?? {
359-
inputTokens: 0,
360-
outputTokens: 0,
361-
cacheCreationTokens: 0,
362-
cacheReadTokens: 0,
363-
cost: 0,
364-
};
365-
366-
modelAggregates.set(modelName, {
367-
inputTokens: existing.inputTokens + (entry.data.message.usage.input_tokens ?? 0),
368-
outputTokens: existing.outputTokens + (entry.data.message.usage.output_tokens ?? 0),
369-
cacheCreationTokens: existing.cacheCreationTokens + (entry.data.message.usage.cache_creation_input_tokens ?? 0),
370-
cacheReadTokens: existing.cacheReadTokens + (entry.data.message.usage.cache_read_input_tokens ?? 0),
371-
cost: existing.cost + entry.cost,
372-
});
373-
}
352+
const modelAggregates = aggregateByModel(
353+
entries,
354+
entry => entry.model,
355+
entry => entry.data.message.usage,
356+
entry => entry.cost,
357+
);
374358

375359
// Create model breakdowns
376-
const modelBreakdowns: ModelBreakdown[] = Array.from(modelAggregates.entries())
377-
.map(([modelName, stats]) => ({
378-
modelName,
379-
...stats,
380-
}))
381-
.sort((a, b) => b.cost - a.cost); // Sort by cost descending
360+
const modelBreakdowns = createModelBreakdowns(modelAggregates);
382361

383362
// Calculate totals
384-
const totals = entries.reduce(
385-
(acc, entry) => ({
386-
inputTokens:
387-
acc.inputTokens + (entry.data.message.usage.input_tokens ?? 0),
388-
outputTokens:
389-
acc.outputTokens + (entry.data.message.usage.output_tokens ?? 0),
390-
cacheCreationTokens:
391-
acc.cacheCreationTokens
392-
+ (entry.data.message.usage.cache_creation_input_tokens ?? 0),
393-
cacheReadTokens:
394-
acc.cacheReadTokens
395-
+ (entry.data.message.usage.cache_read_input_tokens ?? 0),
396-
totalCost: acc.totalCost + entry.cost,
397-
}),
398-
{
399-
inputTokens: 0,
400-
outputTokens: 0,
401-
cacheCreationTokens: 0,
402-
cacheReadTokens: 0,
403-
totalCost: 0,
404-
},
363+
const totals = calculateTotals(
364+
entries,
365+
entry => entry.data.message.usage,
366+
entry => entry.cost,
405367
);
406368

407-
const modelsUsed = [...new Set(entries.map(e => e.model).filter((m): m is string => m != null && m !== '<synthetic>'))];
369+
const modelsUsed = extractUniqueModels(entries, e => e.model);
408370

409371
return {
410372
date,
@@ -413,23 +375,13 @@ export async function loadDailyUsageData(
413375
modelBreakdowns,
414376
};
415377
})
416-
.filter(item => item != null)
417-
.filter((item) => {
418-
// Filter by date range if specified
419-
if (options?.since != null || options?.until != null) {
420-
const dateStr = item.date.replace(/-/g, ''); // Convert to YYYYMMDD
421-
if (options.since != null && dateStr < options.since) {
422-
return false;
423-
}
424-
if (options.until != null && dateStr > options.until) {
425-
return false;
426-
}
427-
}
428-
return true;
429-
});
378+
.filter(item => item != null);
379+
380+
// Filter by date range if specified
381+
const filtered = filterByDateRange(results, item => item.date, options?.since, options?.until);
430382

431383
// Sort by date based on order option (default to descending)
432-
return sortByDate(results, item => item.date, options?.order);
384+
return sortByDate(filtered, item => item.date, options?.order);
433385
}
434386

435387
export async function loadSessionData(
@@ -555,70 +507,24 @@ export async function loadSessionData(
555507
}
556508

557509
// Aggregate by model
558-
const modelAggregates = new Map<string, {
559-
inputTokens: number;
560-
outputTokens: number;
561-
cacheCreationTokens: number;
562-
cacheReadTokens: number;
563-
cost: number;
564-
}>();
565-
566-
for (const entry of entries) {
567-
const modelName = entry.model ?? 'unknown';
568-
// Skip synthetic model
569-
if (modelName === '<synthetic>') {
570-
continue;
571-
}
572-
const existing = modelAggregates.get(modelName) ?? {
573-
inputTokens: 0,
574-
outputTokens: 0,
575-
cacheCreationTokens: 0,
576-
cacheReadTokens: 0,
577-
cost: 0,
578-
};
579-
580-
modelAggregates.set(modelName, {
581-
inputTokens: existing.inputTokens + (entry.data.message.usage.input_tokens ?? 0),
582-
outputTokens: existing.outputTokens + (entry.data.message.usage.output_tokens ?? 0),
583-
cacheCreationTokens: existing.cacheCreationTokens + (entry.data.message.usage.cache_creation_input_tokens ?? 0),
584-
cacheReadTokens: existing.cacheReadTokens + (entry.data.message.usage.cache_read_input_tokens ?? 0),
585-
cost: existing.cost + entry.cost,
586-
});
587-
}
510+
const modelAggregates = aggregateByModel(
511+
entries,
512+
entry => entry.model,
513+
entry => entry.data.message.usage,
514+
entry => entry.cost,
515+
);
588516

589517
// Create model breakdowns
590-
const modelBreakdowns: ModelBreakdown[] = Array.from(modelAggregates.entries())
591-
.map(([modelName, stats]) => ({
592-
modelName,
593-
...stats,
594-
}))
595-
.sort((a, b) => b.cost - a.cost);
518+
const modelBreakdowns = createModelBreakdowns(modelAggregates);
596519

597520
// Calculate totals
598-
const totals = entries.reduce(
599-
(acc, entry) => ({
600-
inputTokens:
601-
acc.inputTokens + (entry.data.message.usage.input_tokens ?? 0),
602-
outputTokens:
603-
acc.outputTokens + (entry.data.message.usage.output_tokens ?? 0),
604-
cacheCreationTokens:
605-
acc.cacheCreationTokens
606-
+ (entry.data.message.usage.cache_creation_input_tokens ?? 0),
607-
cacheReadTokens:
608-
acc.cacheReadTokens
609-
+ (entry.data.message.usage.cache_read_input_tokens ?? 0),
610-
totalCost: acc.totalCost + entry.cost,
611-
}),
612-
{
613-
inputTokens: 0,
614-
outputTokens: 0,
615-
cacheCreationTokens: 0,
616-
cacheReadTokens: 0,
617-
totalCost: 0,
618-
},
521+
const totals = calculateTotals(
522+
entries,
523+
entry => entry.data.message.usage,
524+
entry => entry.cost,
619525
);
620526

621-
const modelsUsed = [...new Set(entries.map(e => e.model).filter((m): m is string => m != null && m !== '<synthetic>'))];
527+
const modelsUsed = extractUniqueModels(entries, e => e.model);
622528

623529
return {
624530
sessionId: latestEntry.sessionId,
@@ -630,22 +536,12 @@ export async function loadSessionData(
630536
modelBreakdowns,
631537
};
632538
})
633-
.filter(item => item != null)
634-
.filter((item) => {
635-
// Filter by date range if specified
636-
if (options?.since != null || options?.until != null) {
637-
const dateStr = item.lastActivity.replace(/-/g, ''); // Convert to YYYYMMDD
638-
if (options.since != null && dateStr < options.since) {
639-
return false;
640-
}
641-
if (options.until != null && dateStr > options.until) {
642-
return false;
643-
}
644-
}
645-
return true;
646-
});
539+
.filter(item => item != null);
647540

648-
return sortByDate(results, item => item.lastActivity, options?.order);
541+
// Filter by date range if specified
542+
const filtered = filterByDateRange(results, item => item.lastActivity, options?.since, options?.until);
543+
544+
return sortByDate(filtered, item => item.lastActivity, options?.order);
649545
}
650546

651547
export async function loadMonthlyUsageData(
@@ -665,45 +561,11 @@ export async function loadMonthlyUsageData(
665561
}
666562

667563
// Aggregate model breakdowns across all days
668-
const modelAggregates = new Map<string, {
669-
inputTokens: number;
670-
outputTokens: number;
671-
cacheCreationTokens: number;
672-
cacheReadTokens: number;
673-
cost: number;
674-
}>();
675-
676-
for (const daily of dailyEntries) {
677-
for (const breakdown of daily.modelBreakdowns) {
678-
// Skip synthetic model
679-
if (breakdown.modelName === '<synthetic>') {
680-
continue;
681-
}
682-
const existing = modelAggregates.get(breakdown.modelName) ?? {
683-
inputTokens: 0,
684-
outputTokens: 0,
685-
cacheCreationTokens: 0,
686-
cacheReadTokens: 0,
687-
cost: 0,
688-
};
689-
690-
modelAggregates.set(breakdown.modelName, {
691-
inputTokens: existing.inputTokens + breakdown.inputTokens,
692-
outputTokens: existing.outputTokens + breakdown.outputTokens,
693-
cacheCreationTokens: existing.cacheCreationTokens + breakdown.cacheCreationTokens,
694-
cacheReadTokens: existing.cacheReadTokens + breakdown.cacheReadTokens,
695-
cost: existing.cost + breakdown.cost,
696-
});
697-
}
698-
}
564+
const allBreakdowns = dailyEntries.flatMap(daily => daily.modelBreakdowns);
565+
const modelAggregates = aggregateModelBreakdowns(allBreakdowns);
699566

700567
// Create model breakdowns
701-
const modelBreakdowns: ModelBreakdown[] = Array.from(modelAggregates.entries())
702-
.map(([modelName, stats]) => ({
703-
modelName,
704-
...stats,
705-
}))
706-
.sort((a, b) => b.cost - a.cost);
568+
const modelBreakdowns = createModelBreakdowns(modelAggregates);
707569

708570
// Collect unique models
709571
const modelsSet = new Set<string>();

0 commit comments

Comments
 (0)