Skip to content

Commit 3e2acec

Browse files
committed
fix(server): import long transcripts instead of skipping them
The importer keeps only the first prompt and the newest 200 messages, but readTranscript charged every retained record to the 32 MiB selected-history budget and rejected the whole file once the total passed it. Sessions of roughly 15k+ records were skipped silently, even with little visible text, because the budget mostly counts per-record overhead. Pin the opening exchange (through the first reply to a visible prompt) and keep later records in a window that drops its oldest entries when the budget would overflow. Transcripts under the budget yield the same records as before.
1 parent a01c48f commit 3e2acec

2 files changed

Lines changed: 140 additions & 9 deletions

File tree

‎apps/server/src/project/AgentSessionScanner.test.ts‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1981,6 +1981,85 @@ it.layer(NodeServices.layer)("AgentSessionScanner", (it) => {
19811981
}),
19821982
);
19831983

1984+
it.effect.each(["claudeAgent", "codex"] as const)(
1985+
"imports the newest %s history when a transcript exceeds the budget",
1986+
(source) =>
1987+
Effect.gen(function* () {
1988+
const path = yield* Path.Path;
1989+
const nowMs = Date.parse("2026-08-24T12:00:00.000Z");
1990+
yield* TestClock.setTime(nowMs);
1991+
const claudeHomePath = yield* makeTempDir("t3code-long-claude-");
1992+
const codexHomePath = yield* makeTempDir("t3code-long-codex-");
1993+
const workspace = yield* makeTempDir("t3code-long-workspace-");
1994+
const record = (role: "user" | "assistant", text: string) =>
1995+
encodeTranscriptRecord(
1996+
source === "claudeAgent"
1997+
? {
1998+
type: role,
1999+
sessionId: "long-session",
2000+
cwd: workspace,
2001+
message: { role, content: text },
2002+
}
2003+
: role === "user"
2004+
? { type: "event_msg", payload: { type: "user_message", message: text } }
2005+
: {
2006+
type: "response_item",
2007+
payload: { type: "message", role, content: [{ type: "output_text", text }] },
2008+
},
2009+
);
2010+
// Resumed Codex threads can open with an empty assistant record before the first prompt.
2011+
const header =
2012+
source === "claudeAgent"
2013+
? []
2014+
: [
2015+
encodeTranscriptRecord({
2016+
type: "session_meta",
2017+
payload: { id: "long-session", cwd: workspace },
2018+
}),
2019+
record("assistant", ""),
2020+
];
2021+
// Each reply selects about 2 MiB, so together they exceed the 32 MiB history budget.
2022+
const replies = Array.from({ length: 20 }, (_, index) =>
2023+
record("assistant", `reply ${index} `.padEnd(1024 * 1024, "x")),
2024+
);
2025+
yield* writeTranscript({
2026+
filePath:
2027+
source === "claudeAgent"
2028+
? path.join(claudeHomePath, "projects", "-slug", "long-session.jsonl")
2029+
: path.join(codexHomePath, "sessions", "2026", "08", "24", "rollout-long.jsonl"),
2030+
contents: [
2031+
...header,
2032+
record("user", "First prompt"),
2033+
...replies,
2034+
record("user", "Latest prompt"),
2035+
record("assistant", "Latest reply"),
2036+
].join("\n"),
2037+
mtimeMs: nowMs,
2038+
});
2039+
2040+
const outcomes = yield* runRecentThreadOutcomes({
2041+
claudeHomePath,
2042+
codexHomePath,
2043+
workspaceRoot: workspace,
2044+
});
2045+
2046+
expect(outcomes).toMatchObject([
2047+
{
2048+
_tag: "Importable",
2049+
thread: { providerSessionId: "long-session", title: "First prompt" },
2050+
},
2051+
]);
2052+
const texts =
2053+
outcomes[0]?._tag === "Importable"
2054+
? outcomes[0].thread.messages.map((message) => message.text.slice(0, 12))
2055+
: [];
2056+
expect(texts[0]).toBe("First prompt");
2057+
expect(texts.slice(-2)).toEqual(["Latest promp", "Latest reply"]);
2058+
expect(texts).not.toContain("reply 1 xxxx");
2059+
expect(texts).toContain("reply 19 xxx");
2060+
}),
2061+
);
2062+
19842063
it.effect("reports stat, read, and parse failures as skipped", () =>
19852064
Effect.gen(function* () {
19862065
const path = yield* Path.Path;

‎apps/server/src/project/AgentSessionScanner.ts‎

Lines changed: 61 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ import * as DateTime from "effect/DateTime";
3333
import * as Effect from "effect/Effect";
3434
import * as FileSystem from "effect/FileSystem";
3535
import * as Layer from "effect/Layer";
36+
import * as MutableList from "effect/MutableList";
3637
import * as Option from "effect/Option";
3738
import * as Path from "effect/Path";
3839
import * as Schema from "effect/Schema";
@@ -511,6 +512,30 @@ function extractDecodedCwd(record: DecodedTranscriptRecord): string | null {
511512
return cwd && cwd.length > 0 ? cwd : null;
512513
}
513514

515+
/** Whether a record is a visible user prompt or an assistant reply, using the parser's rules. */
516+
function conversationRole(
517+
source: AgentSessionSource,
518+
record: DecodedTranscriptRecord,
519+
): "user" | "assistant" | null {
520+
if (source === "claudeAgent") {
521+
if (record.isSidechain === true || record.isMeta === true || record.isCompactSummary === true) {
522+
return null;
523+
}
524+
if (record.type === "assistant") return "assistant";
525+
return record.type === "user" && extractText(record.message?.content).length > 0
526+
? "user"
527+
: null;
528+
}
529+
if (record.type === "event_msg" && record.payload?.type === "user_message") {
530+
return record.payload.message?.trim() ? "user" : null;
531+
}
532+
if (record.type !== "response_item" || record.payload?.type !== "message") return null;
533+
if (record.payload.role === "assistant") return "assistant";
534+
return record.payload.role === "user" && extractText(record.payload.content).length > 0
535+
? "user"
536+
: null;
537+
}
538+
514539
function shouldRetainDecodedRecord(
515540
source: AgentSessionSource,
516541
record: DecodedTranscriptRecord,
@@ -811,8 +836,10 @@ export const make = Effect.gen(function* () {
811836

812837
/**
813838
* Project history fields while reading, before allocating whole JSON records.
814-
* Check the file identity on both sides of the read. A selected-history budget
815-
* failure rejects the entire transcript before any imported messages persist.
839+
* Check the file identity on both sides of the read. Only the first prompt and
840+
* the newest messages are imported, so once selected history reaches its
841+
* budget, the oldest records after the opening exchange are dropped. A
842+
* transcript is rejected only when its opening exchange cannot fit.
816843
*/
817844
const readTranscript = Effect.fn("AgentSessionScanner.readTranscript")(function* (
818845
filePath: string,
@@ -829,17 +856,29 @@ export const make = Effect.gen(function* () {
829856
if (!sameTranscriptIdentity(expected, transcriptIdentity(filePath, yield* file.stat))) {
830857
return null;
831858
}
832-
const records: Array<DecodedTranscriptRecord> = [];
859+
// The opening exchange, through the first reply to a visible prompt,
860+
// holds the first prompt, cwd, and Codex session ID.
861+
const opening: Array<DecodedTranscriptRecord> = [];
862+
const recent = MutableList.make<{
863+
readonly record: DecodedTranscriptRecord;
864+
readonly bytes: number;
865+
}>();
866+
let promptSeen = false;
867+
let openingComplete = false;
833868
let historyBytes = 0;
834869
let recordBytes = 0;
835870
let recordCount = 0;
836871
let bytesRead = 0;
837872
const reserve = (bytes: number) => {
838873
recordBytes += bytes;
839-
if (historyBytes + recordBytes > MAX_IMPORT_HISTORY_BYTES) {
840-
throw new TranscriptJsonLimitError(
841-
"Transcript selected history exceeds the 32 MiB memory budget",
842-
);
874+
while (historyBytes + recordBytes > MAX_IMPORT_HISTORY_BYTES) {
875+
const oldest = MutableList.take(recent);
876+
if (oldest === MutableList.Empty) {
877+
throw new TranscriptJsonLimitError(
878+
"Transcript selected history exceeds the 32 MiB memory budget",
879+
);
880+
}
881+
historyBytes -= oldest.bytes;
843882
}
844883
};
845884
let reader = createTranscriptJsonReader(reserve, selectTranscriptPath);
@@ -852,7 +891,14 @@ export const make = Effect.gen(function* () {
852891
if (recordCount > recordLimit) return false;
853892
const decoded = decodeTranscriptValue(reader.finish());
854893
if (Option.isSome(decoded) && shouldRetainDecodedRecord(source, decoded.value)) {
855-
records.push(decoded.value);
894+
if (openingComplete) {
895+
MutableList.append(recent, { record: decoded.value, bytes: recordBytes });
896+
} else {
897+
opening.push(decoded.value);
898+
}
899+
const role = conversationRole(source, decoded.value);
900+
promptSeen ||= role === "user";
901+
openingComplete ||= promptSeen && role === "assistant";
856902
historyBytes += recordBytes;
857903
}
858904
recordBytes = 0;
@@ -889,7 +935,13 @@ export const make = Effect.gen(function* () {
889935

890936
if (recordStarted && !(yield* Effect.try(finishRecord))) return null;
891937
return sameTranscriptIdentity(expected, transcriptIdentity(filePath, yield* file.stat))
892-
? { records, recordCount }
938+
? {
939+
records: [
940+
...opening,
941+
...MutableList.toArray(recent).map((entry) => entry.record),
942+
],
943+
recordCount,
944+
}
893945
: null;
894946
}),
895947
),

0 commit comments

Comments
 (0)