@@ -14,7 +14,6 @@ import {
1414import * as HostProcess from "@t3tools/shared/HostProcess" ;
1515import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient" ;
1616import * as Effect from "effect/Effect" ;
17- import * as Exit from "effect/Exit" ;
1817import * as FileSystem from "effect/FileSystem" ;
1918import * as Option from "effect/Option" ;
2019import * as Path from "effect/Path" ;
@@ -94,14 +93,29 @@ export interface OpenCodeUsageReadResult {
9493const isNotFound = ( cause : PlatformError . PlatformError ) => cause . reason . _tag === "NotFound" ;
9594const MESSAGE_PAGE_SIZE = 64 ;
9695
96+ /** Whether the path is itself a symlink, without following it. */
97+ const isSymbolicLink = Effect . fnUntraced ( function * ( path : string ) {
98+ const fileSystem = yield * FileSystem . FileSystem ;
99+ return yield * Effect . isSuccess ( fileSystem . readLink ( path ) ) ;
100+ } ) ;
101+
97102/** Stat of a regular file or directory; a symlink reports as itself and is never followed. */
98103const entryInfo = Effect . fn ( "entryInfo" ) ( function * ( path : string ) {
99104 const fileSystem = yield * FileSystem . FileSystem ;
100- const link = yield * Effect . exit ( fileSystem . readLink ( path ) ) ;
101- if ( Exit . isSuccess ( link ) ) return { type : "SymbolicLink" as const , mtime : Option . none < Date > ( ) } ;
105+ if ( yield * isSymbolicLink ( path ) )
106+ return { type : "SymbolicLink" as const , mtime : Option . none < Date > ( ) } ;
102107 return yield * fileSystem . stat ( path ) ;
103108} ) ;
104109
110+ /**
111+ * Stats run in the libuv thread pool, so a few at a time cut the walk's wall
112+ * time without lengthening any event-loop turn.
113+ */
114+ const LEGACY_STAT_CONCURRENCY = 4 ;
115+
116+ /** Entries stat'd before any is processed, so one huge directory is not held in memory at once. */
117+ const LEGACY_STAT_BATCH = 256 ;
118+
105119/** Reads current SQLite and pre-migration JSON stores without modifying either. */
106120export const readOpenCodeUsage = Effect . fn ( "readOpenCodeUsage" ) ( function * (
107121 root : string ,
@@ -234,34 +248,61 @@ export const readOpenCodeUsage = Effect.fn("readOpenCodeUsage")(function* (
234248 while ( directories . length > 0 ) {
235249 const directory = directories . pop ( ) ! ;
236250 yield * Effect . gen ( function * ( ) {
237- for ( const name of yield * fileSystem . readDirectory ( directory ) ) {
238- const entry = path . join ( directory , name ) ;
239- const info = yield * entryInfo ( entry ) . pipe (
240- Effect . catchTags ( {
241- PlatformError : ( cause ) =>
242- isNotFound ( cause ) ? Effect . succeed ( null ) : Effect . fail ( cause ) ,
243- } ) ,
251+ const names = yield * fileSystem . readDirectory ( directory ) ;
252+ for ( let start = 0 ; start < names . length ; start += LEGACY_STAT_BATCH ) {
253+ const batch = names . slice ( start , start + LEGACY_STAT_BATCH ) ;
254+ const entries = batch . map ( ( name ) => path . join ( directory , name ) ) ;
255+ // `stat` follows symlinks and `readLink` fails, expensively, for
256+ // everything else. Stat a batch, then ask whether an entry is a symlink
257+ // only where the answer changes the result.
258+ const infos = yield * Effect . forEach (
259+ entries ,
260+ ( entry ) =>
261+ fileSystem . stat ( entry ) . pipe (
262+ Effect . catchTags ( {
263+ // A symlink whose target cannot be stat'd is skipped, not an error.
264+ PlatformError : ( cause ) =>
265+ isNotFound ( cause )
266+ ? Effect . succeed ( null )
267+ : isSymbolicLink ( entry ) . pipe (
268+ Effect . flatMap ( ( link ) =>
269+ link ? Effect . succeed ( null ) : Effect . fail ( cause ) ,
270+ ) ,
271+ ) ,
272+ } ) ,
273+ Effect . exit ,
274+ ) ,
275+ { concurrency : LEGACY_STAT_CONCURRENCY } ,
244276 ) ;
245- if ( info ?. type === "Directory" ) {
246- directories . push ( entry ) ;
247- } else if ( info ?. type === "File" && name . endsWith ( ".json" ) ) {
248- found = true ;
249- const id = name . slice ( 0 , - 5 ) ;
250- if ( seen . has ( `opencode:${ id } ` ) ) continue ;
251- // A message cannot be created after its file was last written, so a
252- // file untouched since the window opened holds nothing in range.
253- if ( Option . exists ( info . mtime , ( mtime ) => mtime . getTime ( ) < sinceMs ) ) continue ;
254- const file = { path : entry , records : [ ] as UsageRecord [ ] } ;
255- files . push ( file ) ;
256- yield * fileSystem . readFileString ( entry ) . pipe (
257- Effect . map ( ( source ) => append ( file . records , parseOpenCodeMessage ( source , { id } ) ) ) ,
258- Effect . catchTags ( {
259- PlatformError : ( cause ) => {
260- if ( ! isNotFound ( cause ) ) error = true ;
261- return Effect . void ;
262- } ,
263- } ) ,
264- ) ;
277+ for ( const [ index , name ] of batch . entries ( ) ) {
278+ const entry = entries [ index ] ! ;
279+ const info = yield * infos [ index ] ! ;
280+ if ( info ?. type === "Directory" ) {
281+ if ( ! ( yield * isSymbolicLink ( entry ) ) ) directories . push ( entry ) ;
282+ } else if ( info ?. type === "File" && name . endsWith ( ".json" ) ) {
283+ const id = name . slice ( 0 , - 5 ) ;
284+ // A message cannot be created after its file was last written, so a
285+ // file untouched since the window opened holds nothing in range.
286+ const skip =
287+ seen . has ( `opencode:${ id } ` ) ||
288+ Option . exists ( info . mtime , ( mtime ) => mtime . getTime ( ) < sinceMs ) ;
289+ // A skipped file only matters as proof that the store exists.
290+ if ( found && skip ) continue ;
291+ if ( yield * isSymbolicLink ( entry ) ) continue ;
292+ found = true ;
293+ if ( skip ) continue ;
294+ const file = { path : entry , records : [ ] as UsageRecord [ ] } ;
295+ files . push ( file ) ;
296+ yield * fileSystem . readFileString ( entry ) . pipe (
297+ Effect . map ( ( source ) => append ( file . records , parseOpenCodeMessage ( source , { id } ) ) ) ,
298+ Effect . catchTags ( {
299+ PlatformError : ( cause ) => {
300+ if ( ! isNotFound ( cause ) ) error = true ;
301+ return Effect . void ;
302+ } ,
303+ } ) ,
304+ ) ;
305+ }
265306 }
266307 }
267308 } ) . pipe (
0 commit comments