Skip to content

Commit 9b6dde9

Browse files
fix(server): session import finds compressed Codex rollouts (#34947)
1 parent 759e649 commit 9b6dde9

2 files changed

Lines changed: 166 additions & 9 deletions

File tree

‎apps/server/src/project/AgentSessionScanner.test.ts‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import * as NodeChildProcess from "node:child_process";
55
import * as NodeFS from "node:fs";
66
import * as NodeOS from "node:os";
77
import * as NodeSqlite from "node:sqlite";
8+
import * as NodeZlib from "node:zlib";
89
import { describe, expect, it } from "@effect/vitest";
910
import {
1011
AgentSessionImportSource,
@@ -174,6 +175,21 @@ const writeTranscript = Effect.fn("AgentSessionScanner.test.writeTranscript")(fu
174175
yield* fileSystem.utimes(input.filePath, seconds, seconds);
175176
});
176177

178+
const writeCompressedTranscript = Effect.fn("AgentSessionScanner.test.writeCompressedTranscript")(
179+
function* (input: {
180+
readonly filePath: string;
181+
readonly contents: string;
182+
readonly mtimeMs: number;
183+
}) {
184+
const fileSystem = yield* FileSystem.FileSystem;
185+
const path = yield* Path.Path;
186+
yield* fileSystem.makeDirectory(path.dirname(input.filePath), { recursive: true });
187+
yield* fileSystem.writeFile(input.filePath, NodeZlib.zstdCompressSync(input.contents));
188+
const seconds = input.mtimeMs / 1000;
189+
yield* fileSystem.utimes(input.filePath, seconds, seconds);
190+
},
191+
);
192+
177193
const claudeSessionLine = (cwd: string) =>
178194
`${JSON.stringify({ type: "user", cwd, sessionId: "s1" })}\n${JSON.stringify({ type: "assistant" })}\n`;
179195

@@ -1244,6 +1260,32 @@ it.layer(NodeServices.layer)("AgentSessionScanner", (it) => {
12441260

12451261
const result = yield* runScan({ claudeHomePath, codexHomePath });
12461262

1263+
expect(result.candidates.map(({ path, threadCount }) => ({ path, threadCount }))).toEqual([
1264+
{ path: workspace, threadCount: 1 },
1265+
]);
1266+
}),
1267+
);
1268+
it.effect("discovers projects whose Codex rollouts are zstd-compressed", () =>
1269+
Effect.gen(function* () {
1270+
const path = yield* Path.Path;
1271+
const claudeHomePath = yield* makeTempDir("t3code-claude-home-");
1272+
const codexHomePath = yield* makeTempDir("t3code-codex-home-");
1273+
const workspace = yield* makeTempDir("t3code-workspace-");
1274+
yield* writeCompressedTranscript({
1275+
filePath: path.join(
1276+
codexHomePath,
1277+
"sessions",
1278+
"2026",
1279+
"05",
1280+
"03",
1281+
"rollout-2026-05-03T00-00-00-019a5f00-0000-7000-8000-00000000000b.jsonl.zst",
1282+
),
1283+
contents: codexRolloutLine(workspace),
1284+
mtimeMs: Date.parse("2026-05-03T00:00:00.000Z"),
1285+
});
1286+
1287+
const result = yield* runScan({ claudeHomePath, codexHomePath });
1288+
12471289
expect(result.candidates.map(({ path, threadCount }) => ({ path, threadCount }))).toEqual([
12481290
{ path: workspace, threadCount: 1 },
12491291
]);
@@ -1252,6 +1294,66 @@ it.layer(NodeServices.layer)("AgentSessionScanner", (it) => {
12521294
});
12531295

12541296
describe("recentThreads", () => {
1297+
it.effect("imports Codex conversations stored only as zstd-compressed rollouts", () =>
1298+
Effect.gen(function* () {
1299+
const path = yield* Path.Path;
1300+
const nowMs = Date.parse("2026-08-24T12:00:00.000Z");
1301+
yield* TestClock.setTime(nowMs);
1302+
const claudeHomePath = yield* makeTempDir("t3code-claude-home-");
1303+
const codexHomePath = yield* makeTempDir("t3code-codex-home-");
1304+
const workspace = yield* makeTempDir("t3code-workspace-");
1305+
yield* writeCompressedTranscript({
1306+
filePath: path.join(
1307+
codexHomePath,
1308+
"sessions",
1309+
"2026",
1310+
"08",
1311+
"20",
1312+
"rollout-2026-08-20T10-00-00-019a5f00-0000-7000-8000-00000000000c.jsonl.zst",
1313+
),
1314+
contents: [
1315+
encodeTranscriptRecord({
1316+
type: "session_meta",
1317+
payload: { id: "019a5f00-0000-7000-8000-00000000000c", cwd: workspace },
1318+
}),
1319+
encodeTranscriptRecord({
1320+
type: "event_msg",
1321+
timestamp: "2026-08-20T10:00:00.000Z",
1322+
payload: { type: "user_message", message: "Compressed prompt" },
1323+
}),
1324+
encodeTranscriptRecord({
1325+
type: "response_item",
1326+
timestamp: "2026-08-20T10:01:00.000Z",
1327+
payload: {
1328+
type: "message",
1329+
role: "assistant",
1330+
content: [{ type: "output_text", text: "Compressed answer" }],
1331+
},
1332+
}),
1333+
].join("\n"),
1334+
mtimeMs: nowMs - 4 * 24 * 60 * 60 * 1000,
1335+
});
1336+
1337+
const threads = yield* runRecentThreads({
1338+
claudeHomePath,
1339+
codexHomePath,
1340+
workspaceRoot: workspace,
1341+
});
1342+
1343+
expect(
1344+
threads.map((thread) => ({
1345+
id: thread.providerSessionId,
1346+
messages: thread.messages.map((message) => message.text),
1347+
})),
1348+
).toEqual([
1349+
{
1350+
id: "019a5f00-0000-7000-8000-00000000000c",
1351+
messages: ["Compressed prompt", "Compressed answer"],
1352+
},
1353+
]);
1354+
}),
1355+
);
1356+
12551357
it.effect("imports recent Claude, Codex, and Pi sessions for the selected project only", () =>
12561358
Effect.gen(function* () {
12571359
const path = yield* Path.Path;

‎apps/server/src/project/AgentSessionScanner.ts‎

Lines changed: 64 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import * as NodeOS from "node:os";
22
import * as NodeSqlite from "node:sqlite";
3+
import * as NodeZlib from "node:zlib";
34

45
import {
56
AgentSessionImportSource,
@@ -17,11 +18,13 @@ import {
1718
import * as Clock from "effect/Clock";
1819
import * as Context from "effect/Context";
1920
import * as DateTime from "effect/DateTime";
21+
import * as NodeStream from "@effect/platform-node/NodeStream";
2022
import * as Effect from "effect/Effect";
2123
import * as FileSystem from "effect/FileSystem";
2224
import * as Layer from "effect/Layer";
2325
import * as Option from "effect/Option";
2426
import * as Path from "effect/Path";
27+
import * as Pull from "effect/Pull";
2528
import * as Schema from "effect/Schema";
2629
import * as Semaphore from "effect/Semaphore";
2730
import * as Stream from "effect/Stream";
@@ -177,7 +180,11 @@ const decodeCodexHistoryBase = Schema.decodeUnknownOption(
177180
);
178181
// Original rollouts end in their thread ID, rewound ones in `<thread ID>_<rollout ID>`.
179182
const CODEX_ROLLOUT_ID_PATTERN =
180-
/-([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:_([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}))?\.jsonl$/i;
183+
/-([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:_([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}))?\.jsonl(?:\.zst)?$/i;
184+
// Codex zstd-compresses older rollouts; their byte limits and offsets count decompressed bytes.
185+
const COMPRESSED_TRANSCRIPT_SUFFIX = ".jsonl.zst";
186+
const isCompressedTranscript = (filePath: string) =>
187+
filePath.endsWith(COMPRESSED_TRANSCRIPT_SUFFIX);
181188
// Codex records a chat's project only here; moving a chat leaves its rollout cwd unchanged.
182189
const CODEX_STATE_DATABASE = "state_5.sqlite";
183190
const decodeCodexSessionIndexEntry = Schema.decodeUnknownOption(
@@ -861,6 +868,33 @@ export const make = Effect.gen(function* () {
861868
const statOption = (target: string) =>
862869
fileSystem.stat(target).pipe(Effect.asSome, Effect.orElseSucceed(Option.none));
863870

871+
const transcriptChunkReader = Effect.fn("AgentSessionScanner.transcriptChunkReader")(function* (
872+
file: FileSystem.File,
873+
filePath: string,
874+
) {
875+
if (!isCompressedTranscript(filePath)) return (size: number) => file.readAlloc(size);
876+
const pull = yield* Stream.toPull(
877+
fileSystem
878+
.stream(filePath)
879+
.pipe(NodeStream.pipeThroughSimple(() => NodeZlib.createZstdDecompress())),
880+
);
881+
let pending: Array<Uint8Array> = [];
882+
let done = false;
883+
// Untraced: called once per chunk inside the caller's traced read.
884+
return Effect.fnUntraced(function* (size: number) {
885+
while (pending.length === 0) {
886+
if (done) return Option.none<Uint8Array>();
887+
const next = yield* pull.pipe(Pull.catchDone(() => Effect.undefined));
888+
if (next === undefined) done = true;
889+
else pending = [...next];
890+
}
891+
const chunk = pending.shift();
892+
if (chunk === undefined) return Option.none<Uint8Array>();
893+
if (chunk.byteLength > size) pending.unshift(chunk.subarray(size));
894+
return Option.some(chunk.subarray(0, size));
895+
});
896+
});
897+
864898
const directoryIdentity = Effect.fn("AgentSessionScanner.directoryIdentity")(function* (
865899
target: string,
866900
knownStats?: FileSystem.File.Info,
@@ -1037,11 +1071,15 @@ export const make = Effect.gen(function* () {
10371071
fileSystem.open(transcript.filePath, { flag: "r" }).pipe(
10381072
Effect.flatMap((file) =>
10391073
Effect.gen(function* () {
1074+
const readChunk = yield* transcriptChunkReader(file, transcript.filePath);
1075+
const size = isCompressedTranscript(transcript.filePath)
1076+
? Number.POSITIVE_INFINITY
1077+
: transcript.identity.size;
10401078
const decoder = new TextDecoder();
10411079
let remaining = "";
10421080
let bytesRead = 0;
10431081
let recordsRead = 0;
1044-
const maxBytes = Math.min(MAX_TRANSCRIPT_SCAN_BYTES, transcript.identity.size);
1082+
const maxBytes = Math.min(MAX_TRANSCRIPT_SCAN_BYTES, size);
10451083
const reserveRecord = () => {
10461084
if (
10471085
recordsRead === MAX_METADATA_RECORDS_PER_TRANSCRIPT ||
@@ -1073,7 +1111,7 @@ export const make = Effect.gen(function* () {
10731111
);
10741112
budget.operationsRemaining -= 1;
10751113
budget.bytesRemaining -= readSize;
1076-
const next = yield* file.readAlloc(readSize);
1114+
const next = yield* readChunk(readSize);
10771115
if (Option.isNone(next)) {
10781116
return readLastRecord();
10791117
}
@@ -1090,7 +1128,7 @@ export const make = Effect.gen(function* () {
10901128
}
10911129
}
10921130

1093-
if (bytesRead < transcript.identity.size) {
1131+
if (bytesRead < size) {
10941132
budget.truncated = true;
10951133
return null;
10961134
}
@@ -1107,8 +1145,12 @@ export const make = Effect.gen(function* () {
11071145
source: AgentSessionSource,
11081146
prefix?: { readonly endByteOffset: number; readonly historyBytes: number },
11091147
) {
1110-
const readBytes = prefix?.endByteOffset ?? expected.size;
1111-
if (readBytes > expected.size || readBytes > MAX_IMPORTED_TRANSCRIPT_BYTES) return null;
1148+
const compressed = isCompressedTranscript(filePath);
1149+
const readBytes =
1150+
prefix?.endByteOffset ?? (compressed ? MAX_IMPORTED_TRANSCRIPT_BYTES : expected.size);
1151+
if ((!compressed && readBytes > expected.size) || readBytes > MAX_IMPORTED_TRANSCRIPT_BYTES) {
1152+
return null;
1153+
}
11121154

11131155
return yield* Effect.scoped(
11141156
fileSystem.open(filePath, { flag: "r" }).pipe(
@@ -1117,6 +1159,7 @@ export const make = Effect.gen(function* () {
11171159
if (!sameTranscriptIdentity(expected, transcriptIdentity(filePath, yield* file.stat))) {
11181160
return null;
11191161
}
1162+
const readChunk = yield* transcriptChunkReader(file, filePath);
11201163
const records: Array<DecodedTranscriptRecord> = [];
11211164
const claudeHistory = createClaudeHistoryWindow();
11221165
let cwd = Option.none<string>();
@@ -1177,10 +1220,12 @@ export const make = Effect.gen(function* () {
11771220
};
11781221

11791222
while (bytesRead < readBytes) {
1180-
const next = yield* file.readAlloc(
1223+
const next = yield* readChunk(
11811224
Math.min(TRANSCRIPT_PREFIX_BYTES, readBytes - bytesRead),
11821225
);
11831226
if (Option.isNone(next)) {
1227+
// A compressed rollout's decompressed length is only known at its end.
1228+
if (compressed && prefix === undefined) break;
11841229
return null;
11851230
}
11861231

@@ -1398,7 +1443,12 @@ export const make = Effect.gen(function* () {
13981443
}
13991444
const directory = path.join(sessionsDir, year, month, day);
14001445
for (const entry of (yield* readDirectory(directory)).toSorted().toReversed()) {
1401-
if (!entry.startsWith("rollout-") || !entry.endsWith(".jsonl")) continue;
1446+
if (
1447+
!entry.startsWith("rollout-") ||
1448+
!(entry.endsWith(".jsonl") || isCompressedTranscript(entry))
1449+
) {
1450+
continue;
1451+
}
14021452
if (operationsRemaining <= 0) {
14031453
truncated = true;
14041454
break;
@@ -2024,7 +2074,12 @@ export const make = Effect.gen(function* () {
20242074
{
20252075
source: candidate.source,
20262076
providerInstanceId: candidate.providerInstanceId,
2027-
fallbackSessionId: path.basename(transcript.filePath, ".jsonl"),
2077+
fallbackSessionId: path.basename(
2078+
transcript.filePath,
2079+
isCompressedTranscript(transcript.filePath)
2080+
? COMPRESSED_TRANSCRIPT_SUFFIX
2081+
: ".jsonl",
2082+
),
20282083
lastActiveAtMs: transcript.mtimeMs,
20292084
},
20302085
inheritedRecords === null

0 commit comments

Comments
 (0)