@@ -80,9 +80,10 @@ export interface TranscriptParseResult {
8080
8181/** 64 bytes of JSONL tail is ample to distinguish a replaced file. */
8282export const GUARD_LENGTH = 64 ;
83- // Keep native JSON.parse for ordinary records; project large records before
84- // decoding their usage. This is a switch in readers, never a record-size limit.
85- const STREAMING_THRESHOLD_BYTES = 1024 * 1024 ;
83+ // Native parsing is faster for common 1–4 MiB context/tool records. Above
84+ // 8 MiB, project usage without allocating the whole record. This switches
85+ // readers; it never discards a record because of its size.
86+ const STREAMING_THRESHOLD_BYTES = 8 * 1024 * 1024 ;
8687const NEWLINE = 0x0a ;
8788const CARRIAGE_RETURN = 0x0d ;
8889
@@ -252,7 +253,9 @@ export async function readTranscriptRecords(
252253 filePath : string ,
253254 provider : UsageProviderKind ,
254255 resumeFrom ?: TranscriptParsePosition ,
256+ options ?: { readonly streamingThresholdBytes ?: number } ,
255257) : Promise < TranscriptParseResult | null > {
258+ const streamingThresholdBytes = options ?. streamingThresholdBytes ?? STREAMING_THRESHOLD_BYTES ;
256259 let handle : NodeFSP . FileHandle ;
257260 try {
258261 handle = await NodeFSP . open ( filePath , "r" ) ;
@@ -317,7 +320,7 @@ export async function readTranscriptRecords(
317320 const selectPath = selectUsageFields ( provider ) ;
318321
319322 const append = ( segment : Buffer ) => {
320- if ( ! streaming && pendingBytes + segment . length <= STREAMING_THRESHOLD_BYTES ) {
323+ if ( ! streaming && pendingBytes + segment . length <= streamingThresholdBytes ) {
321324 if ( segment . length > 0 ) pendingChunks . push ( segment ) ;
322325 pendingBytes += segment . length ;
323326 return ;
@@ -361,6 +364,7 @@ export async function readTranscriptRecords(
361364 const stream = handle . createReadStream ( {
362365 start,
363366 autoClose : false ,
367+ highWaterMark : 256 * 1024 ,
364368 } ) as AsyncIterable < Buffer > ;
365369 for await ( const chunk of stream ) {
366370 let lineStart = 0 ;
0 commit comments