11import * as NodeOS from "node:os" ;
22import * as NodeSqlite from "node:sqlite" ;
3+ import * as NodeZlib from "node:zlib" ;
34
45import {
56 AgentSessionImportSource ,
@@ -17,11 +18,13 @@ import {
1718import * as Clock from "effect/Clock" ;
1819import * as Context from "effect/Context" ;
1920import * as DateTime from "effect/DateTime" ;
21+ import * as NodeStream from "@effect/platform-node/NodeStream" ;
2022import * as Effect from "effect/Effect" ;
2123import * as FileSystem from "effect/FileSystem" ;
2224import * as Layer from "effect/Layer" ;
2325import * as Option from "effect/Option" ;
2426import * as Path from "effect/Path" ;
27+ import * as Pull from "effect/Pull" ;
2528import * as Schema from "effect/Schema" ;
2629import * as Semaphore from "effect/Semaphore" ;
2730import * as Stream from "effect/Stream" ;
@@ -177,7 +180,11 @@ const decodeCodexHistoryBase = Schema.decodeUnknownOption(
177180) ;
178181// Original rollouts end in their thread ID, rewound ones in `<thread ID>_<rollout ID>`.
179182const CODEX_ROLLOUT_ID_PATTERN =
180- / - ( [ 0 - 9 a - f ] { 8 } - [ 0 - 9 a - f ] { 4 } - [ 0 - 9 a - f ] { 4 } - [ 0 - 9 a - f ] { 4 } - [ 0 - 9 a - f ] { 12 } ) (?: _ ( [ 0 - 9 a - f ] { 8 } - [ 0 - 9 a - f ] { 4 } - [ 0 - 9 a - f ] { 4 } - [ 0 - 9 a - f ] { 4 } - [ 0 - 9 a - f ] { 12 } ) ) ? \. j s o n l $ / i;
183+ / - ( [ 0 - 9 a - f ] { 8 } - [ 0 - 9 a - f ] { 4 } - [ 0 - 9 a - f ] { 4 } - [ 0 - 9 a - f ] { 4 } - [ 0 - 9 a - f ] { 12 } ) (?: _ ( [ 0 - 9 a - f ] { 8 } - [ 0 - 9 a - f ] { 4 } - [ 0 - 9 a - f ] { 4 } - [ 0 - 9 a - f ] { 4 } - [ 0 - 9 a - f ] { 12 } ) ) ? \. j s o n l (?: \. z s t ) ? $ / i;
184+ // Codex zstd-compresses older rollouts; their byte limits and offsets count decompressed bytes.
185+ const COMPRESSED_TRANSCRIPT_SUFFIX = ".jsonl.zst" ;
186+ const isCompressedTranscript = ( filePath : string ) =>
187+ filePath . endsWith ( COMPRESSED_TRANSCRIPT_SUFFIX ) ;
181188// Codex records a chat's project only here; moving a chat leaves its rollout cwd unchanged.
182189const CODEX_STATE_DATABASE = "state_5.sqlite" ;
183190const decodeCodexSessionIndexEntry = Schema . decodeUnknownOption (
@@ -861,6 +868,33 @@ export const make = Effect.gen(function* () {
861868 const statOption = ( target : string ) =>
862869 fileSystem . stat ( target ) . pipe ( Effect . asSome , Effect . orElseSucceed ( Option . none ) ) ;
863870
871+ const transcriptChunkReader = Effect . fn ( "AgentSessionScanner.transcriptChunkReader" ) ( function * (
872+ file : FileSystem . File ,
873+ filePath : string ,
874+ ) {
875+ if ( ! isCompressedTranscript ( filePath ) ) return ( size : number ) => file . readAlloc ( size ) ;
876+ const pull = yield * Stream . toPull (
877+ fileSystem
878+ . stream ( filePath )
879+ . pipe ( NodeStream . pipeThroughSimple ( ( ) => NodeZlib . createZstdDecompress ( ) ) ) ,
880+ ) ;
881+ let pending : Array < Uint8Array > = [ ] ;
882+ let done = false ;
883+ // Untraced: called once per chunk inside the caller's traced read.
884+ return Effect . fnUntraced ( function * ( size : number ) {
885+ while ( pending . length === 0 ) {
886+ if ( done ) return Option . none < Uint8Array > ( ) ;
887+ const next = yield * pull . pipe ( Pull . catchDone ( ( ) => Effect . undefined ) ) ;
888+ if ( next === undefined ) done = true ;
889+ else pending = [ ...next ] ;
890+ }
891+ const chunk = pending . shift ( ) ;
892+ if ( chunk === undefined ) return Option . none < Uint8Array > ( ) ;
893+ if ( chunk . byteLength > size ) pending . unshift ( chunk . subarray ( size ) ) ;
894+ return Option . some ( chunk . subarray ( 0 , size ) ) ;
895+ } ) ;
896+ } ) ;
897+
864898 const directoryIdentity = Effect . fn ( "AgentSessionScanner.directoryIdentity" ) ( function * (
865899 target : string ,
866900 knownStats ?: FileSystem . File . Info ,
@@ -1037,11 +1071,15 @@ export const make = Effect.gen(function* () {
10371071 fileSystem . open ( transcript . filePath , { flag : "r" } ) . pipe (
10381072 Effect . flatMap ( ( file ) =>
10391073 Effect . gen ( function * ( ) {
1074+ const readChunk = yield * transcriptChunkReader ( file , transcript . filePath ) ;
1075+ const size = isCompressedTranscript ( transcript . filePath )
1076+ ? Number . POSITIVE_INFINITY
1077+ : transcript . identity . size ;
10401078 const decoder = new TextDecoder ( ) ;
10411079 let remaining = "" ;
10421080 let bytesRead = 0 ;
10431081 let recordsRead = 0 ;
1044- const maxBytes = Math . min ( MAX_TRANSCRIPT_SCAN_BYTES , transcript . identity . size ) ;
1082+ const maxBytes = Math . min ( MAX_TRANSCRIPT_SCAN_BYTES , size ) ;
10451083 const reserveRecord = ( ) => {
10461084 if (
10471085 recordsRead === MAX_METADATA_RECORDS_PER_TRANSCRIPT ||
@@ -1073,7 +1111,7 @@ export const make = Effect.gen(function* () {
10731111 ) ;
10741112 budget . operationsRemaining -= 1 ;
10751113 budget . bytesRemaining -= readSize ;
1076- const next = yield * file . readAlloc ( readSize ) ;
1114+ const next = yield * readChunk ( readSize ) ;
10771115 if ( Option . isNone ( next ) ) {
10781116 return readLastRecord ( ) ;
10791117 }
@@ -1090,7 +1128,7 @@ export const make = Effect.gen(function* () {
10901128 }
10911129 }
10921130
1093- if ( bytesRead < transcript . identity . size ) {
1131+ if ( bytesRead < size ) {
10941132 budget . truncated = true ;
10951133 return null ;
10961134 }
@@ -1107,8 +1145,12 @@ export const make = Effect.gen(function* () {
11071145 source : AgentSessionSource ,
11081146 prefix ?: { readonly endByteOffset : number ; readonly historyBytes : number } ,
11091147 ) {
1110- const readBytes = prefix ?. endByteOffset ?? expected . size ;
1111- if ( readBytes > expected . size || readBytes > MAX_IMPORTED_TRANSCRIPT_BYTES ) return null ;
1148+ const compressed = isCompressedTranscript ( filePath ) ;
1149+ const readBytes =
1150+ prefix ?. endByteOffset ?? ( compressed ? MAX_IMPORTED_TRANSCRIPT_BYTES : expected . size ) ;
1151+ if ( ( ! compressed && readBytes > expected . size ) || readBytes > MAX_IMPORTED_TRANSCRIPT_BYTES ) {
1152+ return null ;
1153+ }
11121154
11131155 return yield * Effect . scoped (
11141156 fileSystem . open ( filePath , { flag : "r" } ) . pipe (
@@ -1117,6 +1159,7 @@ export const make = Effect.gen(function* () {
11171159 if ( ! sameTranscriptIdentity ( expected , transcriptIdentity ( filePath , yield * file . stat ) ) ) {
11181160 return null ;
11191161 }
1162+ const readChunk = yield * transcriptChunkReader ( file , filePath ) ;
11201163 const records : Array < DecodedTranscriptRecord > = [ ] ;
11211164 const claudeHistory = createClaudeHistoryWindow ( ) ;
11221165 let cwd = Option . none < string > ( ) ;
@@ -1177,10 +1220,12 @@ export const make = Effect.gen(function* () {
11771220 } ;
11781221
11791222 while ( bytesRead < readBytes ) {
1180- const next = yield * file . readAlloc (
1223+ const next = yield * readChunk (
11811224 Math . min ( TRANSCRIPT_PREFIX_BYTES , readBytes - bytesRead ) ,
11821225 ) ;
11831226 if ( Option . isNone ( next ) ) {
1227+ // A compressed rollout's decompressed length is only known at its end.
1228+ if ( compressed && prefix === undefined ) break ;
11841229 return null ;
11851230 }
11861231
@@ -1398,7 +1443,12 @@ export const make = Effect.gen(function* () {
13981443 }
13991444 const directory = path . join ( sessionsDir , year , month , day ) ;
14001445 for ( const entry of ( yield * readDirectory ( directory ) ) . toSorted ( ) . toReversed ( ) ) {
1401- if ( ! entry . startsWith ( "rollout-" ) || ! entry . endsWith ( ".jsonl" ) ) continue ;
1446+ if (
1447+ ! entry . startsWith ( "rollout-" ) ||
1448+ ! ( entry . endsWith ( ".jsonl" ) || isCompressedTranscript ( entry ) )
1449+ ) {
1450+ continue ;
1451+ }
14021452 if ( operationsRemaining <= 0 ) {
14031453 truncated = true ;
14041454 break ;
@@ -2024,7 +2074,12 @@ export const make = Effect.gen(function* () {
20242074 {
20252075 source : candidate . source ,
20262076 providerInstanceId : candidate . providerInstanceId ,
2027- fallbackSessionId : path . basename ( transcript . filePath , ".jsonl" ) ,
2077+ fallbackSessionId : path . basename (
2078+ transcript . filePath ,
2079+ isCompressedTranscript ( transcript . filePath )
2080+ ? COMPRESSED_TRANSCRIPT_SUFFIX
2081+ : ".jsonl" ,
2082+ ) ,
20282083 lastActiveAtMs : transcript . mtimeMs ,
20292084 } ,
20302085 inheritedRecords === null
0 commit comments