Skip to content

Commit 5d38fd0

Browse files
t3dotggclaude
andcommitted
perf(usage): stat walk candidates with a fixed worker pool
Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1 parent 9578991 commit 5d38fd0

1 file changed

Lines changed: 37 additions & 45 deletions

File tree

‎apps/server/src/usage/usageTranscriptReader.ts‎

Lines changed: 37 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -85,8 +85,8 @@ export const GUARD_LENGTH = 64;
8585
// readers; it never discards a record because of its size.
8686
const STREAMING_THRESHOLD_BYTES = 8 * 1024 * 1024;
8787
const NEWLINE = 0x0a;
88-
/** Filesystem calls one transcript walk keeps in flight. The pool has 4 threads. */
89-
const WALK_CONCURRENCY = 32;
88+
/** `stat` calls one transcript walk keeps in flight. The libuv pool has 4 threads. */
89+
const STAT_CONCURRENCY = 32;
9090
const CARRIAGE_RETURN = 0x0d;
9191

9292
type SelectedFields = { readonly [key: string]: true | SelectedFields };
@@ -159,62 +159,54 @@ function fnv1a(buffer: Buffer): number {
159159
* Grok sessions also ship multi-megabyte `chat_history` and `events` logs that
160160
* never carry usage, so the basename filter keeps a cold scan off those files.
161161
*
162-
* Entries are stat'd concurrently: a warm scan stats thousands of files, and
163-
* one at a time each waits its own trip through the thread pool. At most
164-
* `WALK_CONCURRENCY` calls are in flight, so a huge tree cannot queue ahead of
165-
* the server's other filesystem work. Results keep `readdir` order, which the
166-
* aggregator's first-seen dedupe relies on.
162+
* Directories are listed depth-first, one at a time, then the candidates are
163+
* stat'd by a fixed pool of workers: a warm scan stats thousands of files, and
164+
* one at a time each waits its own trip through the thread pool. Results keep
165+
* `readdir` order, which the aggregator's first-seen dedupe relies on.
167166
*/
168167
export async function listTranscriptFiles(
169168
root: string,
170169
sinceMs: number,
171170
options?: { readonly fileName?: string },
172171
): Promise<readonly TranscriptFile[]> {
173172
const fileName = options?.fileName;
174-
let active = 0;
175-
const waiting: Array<() => void> = [];
176-
const limited = async <A>(call: () => Promise<A>): Promise<A> => {
177-
if (active < WALK_CONCURRENCY) active += 1;
178-
// A finished call hands its slot straight to the next waiter.
179-
else await new Promise<void>((resolve) => waiting.push(resolve));
180-
try {
181-
return await call();
182-
} finally {
183-
const next = waiting.shift();
184-
if (next === undefined) active -= 1;
185-
else next();
186-
}
187-
};
188-
189-
const walk = async (dir: string): Promise<readonly TranscriptFile[]> => {
173+
const candidates: string[] = [];
174+
const walk = async (dir: string): Promise<void> => {
190175
let entries;
191176
try {
192-
entries = await limited(() => NodeFSP.readdir(dir, { withFileTypes: true }));
177+
entries = await NodeFSP.readdir(dir, { withFileTypes: true });
193178
} catch {
194-
return [];
179+
return;
180+
}
181+
for (const entry of entries) {
182+
const child = NodePath.join(dir, entry.name);
183+
if (entry.isDirectory()) await walk(child);
184+
else if (fileName !== undefined ? entry.name === fileName : entry.name.endsWith(".jsonl")) {
185+
candidates.push(child);
186+
}
195187
}
196-
const nested = await Promise.all(
197-
entries.map(async (entry): Promise<readonly TranscriptFile[]> => {
198-
const child = NodePath.join(dir, entry.name);
199-
if (entry.isDirectory()) return walk(child);
200-
if (fileName !== undefined ? entry.name !== fileName : !entry.name.endsWith(".jsonl")) {
201-
return [];
202-
}
203-
try {
204-
const stats = await limited(() => NodeFSP.stat(child));
205-
return stats.mtimeMs >= sinceMs
206-
? [{ path: child, size: stats.size, mtimeMs: stats.mtimeMs }]
207-
: [];
208-
} catch {
209-
// Vanished between readdir and stat.
210-
return [];
211-
}
212-
}),
213-
);
214-
return nested.flat();
215188
};
189+
await walk(root);
216190

217-
return walk(root);
191+
const found: Array<TranscriptFile | undefined> = Array.from({ length: candidates.length });
192+
// Each worker pulls the next candidate from one shared iterator.
193+
const queue = candidates.entries();
194+
const statQueued = async (): Promise<void> => {
195+
for (const [index, path] of queue) {
196+
try {
197+
const stats = await NodeFSP.stat(path);
198+
if (stats.mtimeMs >= sinceMs) {
199+
found[index] = { path, size: stats.size, mtimeMs: stats.mtimeMs };
200+
}
201+
} catch {
202+
// Vanished between readdir and stat.
203+
}
204+
}
205+
};
206+
await Promise.all(
207+
Array.from({ length: Math.min(STAT_CONCURRENCY, candidates.length) }, statQueued),
208+
);
209+
return found.filter((file) => file !== undefined);
218210
}
219211

220212
/**

0 commit comments

Comments
 (0)