Skip to content

Commit e4fd685

Browse files
author
jleechan
committed
perf(ccusage): skip Claude session files whose mtime precedes --since
Claude stores sessions as flat UUIDs under projects/<hash>/<uuid>.jsonl with no date directory hierarchy, so the date-tree traversal used for Codex is not applicable. Instead, filter by filesystem mtime after globbing: any file last written more than one day before --since cannot contain entries in the requested range and is skipped before any content is read. The 24-hour buffer absorbs timezone differences between local clock and UTC. Only --since is used for mtime pruning; --until is intentionally ignored because a file updated after --until may still hold entries from before it. Measured on a 28 800-file / 6.7 GB store with --since 7 days ago: Before: ~90 s (all files read via sortFilesByTimestamp) After: ~25 s (10 000 files pass mtime; rest pruned at stat level)
1 parent 809a2ae commit e4fd685

1 file changed

Lines changed: 86 additions & 10 deletions

File tree

‎apps/ccusage/src/data-loader.ts‎

Lines changed: 86 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ import type { LoadedUsageEntry, SessionBlock } from './_session-blocks.ts';
1313
import type { ActivityDate, Bucket, CostMode, ModelName, SortOrder, Version } from './_types.ts';
1414
import { Buffer } from 'node:buffer';
1515
import { createReadStream, createWriteStream } from 'node:fs';
16-
import { readFile } from 'node:fs/promises';
16+
import { readFile, stat } from 'node:fs/promises';
1717
import path from 'node:path';
1818
import process from 'node:process';
1919
import { createInterface } from 'node:readline';
@@ -698,20 +698,64 @@ export type GlobResult = {
698698
};
699699

700700
/**
701-
* Glob files from multiple Claude paths in parallel
701+
* Convert a YYYYMMDD string to a Date at midnight local time.
702+
*/
703+
function parseDateKey(yyyymmdd: string): Date {
704+
const year = Number.parseInt(yyyymmdd.slice(0, 4), 10);
705+
const month = Number.parseInt(yyyymmdd.slice(4, 6), 10) - 1;
706+
const day = Number.parseInt(yyyymmdd.slice(6, 8), 10);
707+
return new Date(year, month, day);
708+
}
709+
710+
/**
711+
* Filter files by filesystem mtime using a 1-day buffer before `since`.
712+
*
713+
* Claude session files have no date directory hierarchy, so we rely on mtime
714+
* (last write time) as a cheap O(stat) proxy. A file whose mtime is more than
715+
* one day before `since` cannot contain entries in [since, ∞) — skip it.
716+
* The 1-day buffer absorbs timezone differences between local clock and UTC.
717+
*
718+
* `until` is intentionally NOT used for mtime pruning: a file updated after
719+
* `until` may still contain entries from before `until`.
720+
*/
721+
async function filterFilesByMtime(files: string[], since: string | undefined): Promise<string[]> {
722+
if (since == null) {
723+
return files;
724+
}
725+
const threshold = parseDateKey(since).getTime() - 24 * 60 * 60 * 1000;
726+
const results = await Promise.all(
727+
files.map(async (file) => {
728+
try {
729+
const { mtimeMs } = await stat(file);
730+
return mtimeMs >= threshold ? file : null;
731+
} catch {
732+
return file; // Include on stat error to avoid silent data loss
733+
}
734+
}),
735+
);
736+
return results.filter((f): f is string => f !== null);
737+
}
738+
739+
/**
740+
* Glob files from multiple Claude paths in parallel, optionally pruning by mtime.
702741
* @param claudePaths - Array of Claude base paths
742+
* @param filter - Optional date range; `since` is used to skip stale files via mtime
703743
* @returns Array of file paths with their base directories
704744
*/
705-
export async function globUsageFiles(claudePaths: string[]): Promise<GlobResult[]> {
745+
export async function globUsageFiles(
746+
claudePaths: string[],
747+
filter?: DateFilter,
748+
): Promise<GlobResult[]> {
706749
const filePromises = claudePaths.map(async (claudePath) => {
707750
const claudeDir = path.join(claudePath, CLAUDE_PROJECTS_DIR_NAME);
708751
const files = await glob([USAGE_DATA_GLOB_PATTERN], {
709752
cwd: claudeDir,
710753
absolute: true,
711754
}).catch(() => []); // Gracefully handle errors for individual paths
712755

756+
const filtered = await filterFilesByMtime(files, filter?.since);
713757
// Map each file to include its base directory
714-
return files.map((file) => ({ file, baseDir: claudeDir }));
758+
return filtered.map((file) => ({ file, baseDir: claudeDir }));
715759
});
716760
return (await Promise.all(filePromises)).flat();
717761
}
@@ -750,8 +794,11 @@ export async function loadDailyUsageData(options?: LoadOptions): Promise<DailyUs
750794
// Get all Claude paths or use the specific one from options
751795
const claudePaths = toArray(options?.claudePath ?? getClaudePaths());
752796

753-
// Collect files from all paths in parallel
754-
const allFiles = await globUsageFiles(claudePaths);
797+
// Collect files from all paths in parallel, pruning by mtime when since is set
798+
const allFiles = await globUsageFiles(claudePaths, {
799+
since: options?.since,
800+
until: options?.until,
801+
});
755802
const fileList = allFiles.map((f) => f.file);
756803

757804
if (fileList.length === 0) {
@@ -899,8 +946,11 @@ export async function loadSessionData(options?: LoadOptions): Promise<SessionUsa
899946
// Get all Claude paths or use the specific one from options
900947
const claudePaths = toArray(options?.claudePath ?? getClaudePaths());
901948

902-
// Collect files from all paths with their base directories in parallel
903-
const filesWithBase = await globUsageFiles(claudePaths);
949+
// Collect files from all paths with their base directories in parallel, pruning by mtime
950+
const filesWithBase = await globUsageFiles(claudePaths, {
951+
since: options?.since,
952+
until: options?.until,
953+
});
904954

905955
if (filesWithBase.length === 0) {
906956
return [];
@@ -1345,15 +1395,16 @@ export async function loadSessionBlockData(options?: LoadOptions): Promise<Sessi
13451395
// Get all Claude paths or use the specific one from options
13461396
const claudePaths = toArray(options?.claudePath ?? getClaudePaths());
13471397

1348-
// Collect files from all paths
1398+
// Collect files from all paths, pruning by mtime when since is set
13491399
const allFiles: string[] = [];
13501400
for (const claudePath of claudePaths) {
13511401
const claudeDir = path.join(claudePath, CLAUDE_PROJECTS_DIR_NAME);
13521402
const files = await glob([USAGE_DATA_GLOB_PATTERN], {
13531403
cwd: claudeDir,
13541404
absolute: true,
13551405
});
1356-
allFiles.push(...files);
1406+
const filtered = await filterFilesByMtime(files, options?.since);
1407+
allFiles.push(...filtered);
13571408
}
13581409

13591410
if (allFiles.length === 0) {
@@ -4691,6 +4742,31 @@ if (import.meta.vitest != null) {
46914742
expect(results).toHaveLength(3);
46924743
expect(results.every((r) => r.baseDir.includes(path.join('path1', 'projects')))).toBe(true);
46934744
});
4745+
4746+
it('skips files whose mtime is more than one day before since', async () => {
4747+
const { utimes } = await import('node:fs/promises');
4748+
4749+
await using fixture = await createFixture({
4750+
'base/projects/proj/old/usage.jsonl': 'old-data',
4751+
'base/projects/proj/new/usage.jsonl': 'new-data',
4752+
});
4753+
4754+
const oldFile = fixture.getPath('base/projects/proj/old/usage.jsonl');
4755+
const newFile = fixture.getPath('base/projects/proj/new/usage.jsonl');
4756+
4757+
// Set old file mtime to 5 days before since, new file mtime to 1 day before since.
4758+
const since = new Date('2026-03-05');
4759+
const oldMtime = new Date(since.getTime() - 5 * 24 * 60 * 60 * 1000);
4760+
const newMtime = new Date(since.getTime() - 12 * 60 * 60 * 1000); // 12h before (within buffer)
4761+
await utimes(oldFile, oldMtime, oldMtime);
4762+
await utimes(newFile, newMtime, newMtime);
4763+
4764+
const results = await globUsageFiles([fixture.getPath('base')], { since: '20260305' });
4765+
4766+
// Old file (5d before since) should be pruned; new file (12h before, within 1d buffer) kept.
4767+
expect(results).toHaveLength(1);
4768+
expect(results[0]!.file).toContain('new');
4769+
});
46944770
});
46954771

46964772
// Test for calculateContextTokens

0 commit comments

Comments
 (0)