@@ -85,8 +85,8 @@ export const GUARD_LENGTH = 64;
8585// readers; it never discards a record because of its size.
8686const STREAMING_THRESHOLD_BYTES = 8 * 1024 * 1024 ;
8787const NEWLINE = 0x0a ;
88- /** Filesystem calls one transcript walk keeps in flight. The pool has 4 threads. */
89- const WALK_CONCURRENCY = 32 ;
88+ /** `stat` calls one transcript walk keeps in flight. The libuv pool has 4 threads. */
89+ const STAT_CONCURRENCY = 32 ;
9090const CARRIAGE_RETURN = 0x0d ;
9191
9292type SelectedFields = { readonly [ key : string ] : true | SelectedFields } ;
@@ -159,62 +159,54 @@ function fnv1a(buffer: Buffer): number {
159159 * Grok sessions also ship multi-megabyte `chat_history` and `events` logs that
160160 * never carry usage, so the basename filter keeps a cold scan off those files.
161161 *
162- * Entries are stat'd concurrently: a warm scan stats thousands of files, and
163- * one at a time each waits its own trip through the thread pool. At most
164- * `WALK_CONCURRENCY` calls are in flight, so a huge tree cannot queue ahead of
165- * the server's other filesystem work. Results keep `readdir` order, which the
166- * aggregator's first-seen dedupe relies on.
162+ * Directories are listed depth-first, one at a time, then the candidates are
163+ * stat'd by a fixed pool of workers: a warm scan stats thousands of files, and
164+ * one at a time each waits its own trip through the thread pool. Results keep
165+ * `readdir` order, which the aggregator's first-seen dedupe relies on.
167166 */
168167export async function listTranscriptFiles (
169168 root : string ,
170169 sinceMs : number ,
171170 options ?: { readonly fileName ?: string } ,
172171) : Promise < readonly TranscriptFile [ ] > {
173172 const fileName = options ?. fileName ;
174- let active = 0 ;
175- const waiting : Array < ( ) => void > = [ ] ;
176- const limited = async < A > ( call : ( ) => Promise < A > ) : Promise < A > => {
177- if ( active < WALK_CONCURRENCY ) active += 1 ;
178- // A finished call hands its slot straight to the next waiter.
179- else await new Promise < void > ( ( resolve ) => waiting . push ( resolve ) ) ;
180- try {
181- return await call ( ) ;
182- } finally {
183- const next = waiting . shift ( ) ;
184- if ( next === undefined ) active -= 1 ;
185- else next ( ) ;
186- }
187- } ;
188-
189- const walk = async ( dir : string ) : Promise < readonly TranscriptFile [ ] > => {
173+ const candidates : string [ ] = [ ] ;
174+ const walk = async ( dir : string ) : Promise < void > => {
190175 let entries ;
191176 try {
192- entries = await limited ( ( ) => NodeFSP . readdir ( dir , { withFileTypes : true } ) ) ;
177+ entries = await NodeFSP . readdir ( dir , { withFileTypes : true } ) ;
193178 } catch {
194- return [ ] ;
179+ return ;
180+ }
181+ for ( const entry of entries ) {
182+ const child = NodePath . join ( dir , entry . name ) ;
183+ if ( entry . isDirectory ( ) ) await walk ( child ) ;
184+ else if ( fileName !== undefined ? entry . name === fileName : entry . name . endsWith ( ".jsonl" ) ) {
185+ candidates . push ( child ) ;
186+ }
195187 }
196- const nested = await Promise . all (
197- entries . map ( async ( entry ) : Promise < readonly TranscriptFile [ ] > => {
198- const child = NodePath . join ( dir , entry . name ) ;
199- if ( entry . isDirectory ( ) ) return walk ( child ) ;
200- if ( fileName !== undefined ? entry . name !== fileName : ! entry . name . endsWith ( ".jsonl" ) ) {
201- return [ ] ;
202- }
203- try {
204- const stats = await limited ( ( ) => NodeFSP . stat ( child ) ) ;
205- return stats . mtimeMs >= sinceMs
206- ? [ { path : child , size : stats . size , mtimeMs : stats . mtimeMs } ]
207- : [ ] ;
208- } catch {
209- // Vanished between readdir and stat.
210- return [ ] ;
211- }
212- } ) ,
213- ) ;
214- return nested . flat ( ) ;
215188 } ;
189+ await walk ( root ) ;
216190
217- return walk ( root ) ;
191+ const found : Array < TranscriptFile | undefined > = Array . from ( { length : candidates . length } ) ;
192+ // Each worker pulls the next candidate from one shared iterator.
193+ const queue = candidates . entries ( ) ;
194+ const statQueued = async ( ) : Promise < void > => {
195+ for ( const [ index , path ] of queue ) {
196+ try {
197+ const stats = await NodeFSP . stat ( path ) ;
198+ if ( stats . mtimeMs >= sinceMs ) {
199+ found [ index ] = { path, size : stats . size , mtimeMs : stats . mtimeMs } ;
200+ }
201+ } catch {
202+ // Vanished between readdir and stat.
203+ }
204+ }
205+ } ;
206+ await Promise . all (
207+ Array . from ( { length : Math . min ( STAT_CONCURRENCY , candidates . length ) } , statQueued ) ,
208+ ) ;
209+ return found . filter ( ( file ) => file !== undefined ) ;
218210}
219211
220212/**
0 commit comments