Skip to content

Commit 6e876b6

Browse files
committed
perf(ccusage): avoid double reading block files
Load session block JSONL files once and collect each file earliest timestamp during the same pass that parses usage entries. This removes the previous sortFilesByTimestamp pre-scan from blocks, which read every file before reading the same files again for aggregation. The standalone getEarliestTimestamp helper now shares the same fast timestamp extraction path, falling back to JSON.parse only for non-compact timestamp lines. Benchmark on Apple M3 Pro, built JavaScript CLI, LOG_LEVEL=0, blocks --offline --json, hyperfine --warmup 1 --runs 5. Process checks showed no hyperfine/mitata/zig benchmark process before the run; cmux and WindowServer were active, so treat the absolute number as local noisy data. JS blocks current: 3.347s ± 0.044s. Previous JS baseline from f5020ef was 6.51s, so this is about 1.95x faster for blocks. Verified with pnpm run format, pnpm typecheck, pnpm --filter ccusage exec vitest run src/data-loader.ts, pnpm --filter ccusage run build, pnpm run test, and JS/Rust token-total parity checks for daily/session/monthly/weekly/blocks.
1 parent f5020ef commit 6e876b6

1 file changed

Lines changed: 61 additions & 21 deletions

File tree

‎apps/ccusage/src/data-loader.ts‎

Lines changed: 61 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,28 @@ function getJSONLFileReadConcurrency(fileCount: number, singleThread = false): n
8787
return Math.max(1, Math.min(fileCount, os.availableParallelism()));
8888
}
8989

90+
function getTimestampFromLine(line: string): Date | null {
91+
const timestamp = extractLastStringField(line, 'timestamp');
92+
if (timestamp != null && ISO_TIMESTAMP_PATTERN.test(timestamp)) {
93+
return new Date(timestamp);
94+
}
95+
96+
if (!line.includes('"timestamp"')) {
97+
return null;
98+
}
99+
100+
try {
101+
const json = JSON.parse(line) as Record<string, unknown>;
102+
if (typeof json.timestamp !== 'string') {
103+
return null;
104+
}
105+
const date = new Date(json.timestamp);
106+
return Number.isNaN(date.getTime()) ? null : date;
107+
} catch {
108+
return null;
109+
}
110+
}
111+
90112
/**
91113
* Get Claude data directories to search for usage data
92114
* When CLAUDE_CONFIG_DIR is set: uses only those paths
@@ -966,18 +988,9 @@ export async function getEarliestTimestamp(filePath: string): Promise<Date | nul
966988
let earliestDate: Date | null = null;
967989

968990
await processJSONLFileByLine(filePath, (line) => {
969-
try {
970-
const json = JSON.parse(line) as Record<string, unknown>;
971-
if (json.timestamp != null && typeof json.timestamp === 'string') {
972-
const date = new Date(json.timestamp);
973-
if (!Number.isNaN(date.getTime())) {
974-
if (earliestDate == null || date < earliestDate) {
975-
earliestDate = date;
976-
}
977-
}
978-
}
979-
} catch {
980-
// Skip invalid JSON lines
991+
const date = getTimestampFromLine(line);
992+
if (date != null && (earliestDate == null || date < earliestDate)) {
993+
earliestDate = date;
981994
}
982995
});
983996

@@ -1206,7 +1219,7 @@ export async function loadDailyUsageData(options?: LoadOptions): Promise<DailyUs
12061219
const fileResults = await mapWithConcurrency(
12071220
mtimeFilteredFiles,
12081221
getJSONLFileReadConcurrency(mtimeFilteredFiles.length, options?.singleThread),
1209-
async (file) => {
1222+
async (file): Promise<DailyEntry[]> => {
12101223
const project = extractProjectFromPath(file);
12111224
const entries: DailyEntry[] = [];
12121225

@@ -1882,9 +1895,6 @@ export async function loadSessionBlockData(options?: LoadOptions): Promise<Sessi
18821895
options?.minUpdateTime,
18831896
);
18841897

1885-
// Sort files by timestamp to ensure chronological processing
1886-
const sortedFiles = await sortFilesByTimestamp(mtimeFilteredFiles);
1887-
18881898
// Fetch pricing data for cost calculation only when needed
18891899
const mode = options?.mode ?? 'auto';
18901900

@@ -1895,17 +1905,33 @@ export async function loadSessionBlockData(options?: LoadOptions): Promise<Sessi
18951905
const allEntries: LoadedUsageEntry[] = [];
18961906
const processedEntries = new Map<string, { data: UsageData; index: number }>();
18971907

1908+
type BlockFileResult = {
1909+
file: string;
1910+
timestamp: Date | null;
1911+
entries: Array<{
1912+
data: UsageData;
1913+
entry: LoadedUsageEntry;
1914+
uniqueHash: string | null;
1915+
}>;
1916+
};
1917+
18981918
const fileResults = await mapWithConcurrency(
1899-
sortedFiles,
1900-
getJSONLFileReadConcurrency(sortedFiles.length, options?.singleThread),
1901-
async (file) => {
1919+
mtimeFilteredFiles,
1920+
getJSONLFileReadConcurrency(mtimeFilteredFiles.length, options?.singleThread),
1921+
async (file): Promise<BlockFileResult> => {
1922+
let timestamp: Date | null = null;
19021923
const entries: Array<{
19031924
data: UsageData;
19041925
entry: LoadedUsageEntry;
19051926
uniqueHash: string | null;
19061927
}> = [];
19071928

19081929
await processJSONLFileByLine(file, async (line) => {
1930+
const lineTimestamp = getTimestampFromLine(line);
1931+
if (lineTimestamp != null && (timestamp == null || lineTimestamp < timestamp)) {
1932+
timestamp = lineTimestamp;
1933+
}
1934+
19091935
let deferred:
19101936
| {
19111937
data: UsageData;
@@ -1961,11 +1987,25 @@ export async function loadSessionBlockData(options?: LoadOptions): Promise<Sessi
19611987
return calculateCostForEntry(deferred.data, mode, fetcher!).then(deferred.pushEntry);
19621988
});
19631989

1964-
return entries;
1990+
return { file, timestamp, entries };
19651991
},
19661992
);
19671993

1968-
for (const entries of fileResults) {
1994+
fileResults.sort((a, b) => {
1995+
if (a.timestamp == null && b.timestamp == null) {
1996+
return a.file.localeCompare(b.file);
1997+
}
1998+
if (a.timestamp == null) {
1999+
return 1;
2000+
}
2001+
if (b.timestamp == null) {
2002+
return -1;
2003+
}
2004+
const timestampDiff = a.timestamp.getTime() - b.timestamp.getTime();
2005+
return timestampDiff === 0 ? a.file.localeCompare(b.file) : timestampDiff;
2006+
});
2007+
2008+
for (const { entries } of fileResults) {
19692009
for (const { data, entry, uniqueHash } of entries) {
19702010
if (uniqueHash == null) {
19712011
allEntries.push(entry);

0 commit comments

Comments
 (0)