Skip to content

Commit 95f9b14

Browse files
authored
fix(server): import transcripts with oversized tool records (pingdotgg#10430)
1 parent 8c9a49a commit 95f9b14

5 files changed

Lines changed: 338 additions & 44 deletions

File tree

‎apps/server/package.json‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@
3131
"effect": "catalog:",
3232
"msgpackr-extract": "3.0.4",
3333
"node-pty": "^1.1.0",
34+
"stream-chain": "^4.2.5",
35+
"stream-json": "3.6.0",
3436
"yaml": "catalog:",
3537
"yauzl": "^3.4.0"
3638
},
Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
import * as SchemaAST from "effect/SchemaAST";
2+
import { isMany, none, type Many } from "stream-chain/defs.js";
3+
import { Assembler } from "stream-json/core/assembler.js";
4+
import { filter } from "stream-json/core/filters/filter.js";
5+
import * as StreamJson from "stream-json/core/parser.js";
6+
import type { ParserOptions, Token } from "stream-json/core/parser.js";
7+
8+
type JsonPath = ReadonlyArray<string | number | null>;
9+
10+
/** Select schema fields before assembling their values, without a second field list. */
11+
export function createTranscriptJsonSelector(schema: { readonly ast: SchemaAST.AST }) {
12+
const ast = SchemaAST.toEncoded(schema.ast);
13+
const includes = (node: SchemaAST.AST, path: JsonPath, index: number): boolean => {
14+
if (index === path.length) return true;
15+
switch (node._tag) {
16+
case "Objects":
17+
// Records have dynamic keys. Keep their values for the decoder to validate.
18+
if (node.indexSignatures.length > 0) return true;
19+
return node.propertySignatures.some(
20+
(property) =>
21+
String(property.name) === path[index] && includes(property.type, path, index + 1),
22+
);
23+
case "Arrays": {
24+
const key = path[index];
25+
if (typeof key !== "number") return true;
26+
const element = node.elements[key];
27+
if (element) return includes(element, path, index + 1);
28+
return node.rest.length === 0 || node.rest.some((item) => includes(item, path, index + 1));
29+
}
30+
case "Union":
31+
return node.types.some((type) => includes(type, path, index));
32+
case "Suspend":
33+
return includes(node.thunk(), path, index);
34+
case "Unknown":
35+
case "Any":
36+
case "ObjectKeyword":
37+
case "Declaration":
38+
// Unstructured/custom schemas must reach the decoder intact. The
39+
// shared budget still bounds their allocations.
40+
return true;
41+
default:
42+
return false;
43+
}
44+
};
45+
return (path: JsonPath) => includes(ast, path, 0);
46+
}
47+
48+
export class TranscriptJsonLimitError extends Error {}
49+
50+
/**
51+
* Project a single JSONL record without materializing unselected string values.
52+
* The caller supplies a shared allocation budget for the entire transcript.
53+
* Budget exhaustion rejects the transcript, never a message within it.
54+
*/
55+
export function createTranscriptJsonReader(
56+
reserve: (bytes: number) => void,
57+
selectPath: (path: JsonPath) => boolean,
58+
) {
59+
// The synchronous tokenizer is exported at runtime in 3.6.0, but omitted
60+
// from its bundled types. Unlike parser(), it does not wrap tokens in an
61+
// async generator; the file reader already supplies backpressure and UTF-8.
62+
const { jsonParser } = StreamJson as typeof StreamJson & {
63+
jsonParser: (
64+
options: ParserOptions,
65+
) => (input: string | typeof none) => Many<Token> | typeof none;
66+
};
67+
const tokenize = jsonParser({ packValues: false });
68+
const select = filter({ filter: selectPath, streamKeys: false }) as (
69+
input: Token | typeof none,
70+
) => Token | Many<Token> | typeof none;
71+
const assembler = new Assembler();
72+
let key: string | null = null;
73+
let value = "";
74+
let depth = 0;
75+
let complete = false;
76+
let malformed = false;
77+
78+
const assemble = (token: Token) => {
79+
reserve(
80+
64 + ("value" in token && typeof token.value === "string" ? token.value.length * 2 : 0),
81+
);
82+
switch (token.name) {
83+
case "startString":
84+
case "startNumber":
85+
value = "";
86+
break;
87+
case "stringChunk":
88+
case "numberChunk":
89+
value += token.value;
90+
break;
91+
case "endString":
92+
assembler.consume({ name: "stringValue", value });
93+
value = "";
94+
break;
95+
case "endNumber":
96+
assembler.consume({ name: "numberValue", value });
97+
value = "";
98+
break;
99+
default:
100+
assembler.consume(token);
101+
}
102+
};
103+
const selectToken = (token: Token | typeof none) => {
104+
const selected = select(token);
105+
if (selected === none) return;
106+
if (isMany(selected)) {
107+
for (const item of selected.values) assemble(item);
108+
} else {
109+
assemble(selected);
110+
}
111+
};
112+
const consume = (input: string | typeof none) => {
113+
if (malformed) return;
114+
try {
115+
const tokens = tokenize(input);
116+
if (tokens === none) return;
117+
for (const token of tokens.values) {
118+
if (token.name === "startObject" || token.name === "startArray") {
119+
if (++depth > 128)
120+
throw new TranscriptJsonLimitError("Transcript JSON nesting exceeds 128 levels");
121+
} else if (token.name === "endObject" || token.name === "endArray") {
122+
if (--depth === 0) complete = true;
123+
}
124+
// Charge keys before assembling them, including unknown names. Reject
125+
// the transcript on exhaustion instead of silently shortening a key.
126+
if (token.name === "startKey") {
127+
key = "";
128+
} else if (token.name === "stringChunk" && key !== null) {
129+
reserve(token.value.length * 2);
130+
key += token.value;
131+
} else if (token.name === "endKey") {
132+
selectToken({ name: "keyValue", value: key ?? "" });
133+
key = null;
134+
} else {
135+
selectToken(token);
136+
}
137+
}
138+
} catch (cause) {
139+
if (cause instanceof Error && cause.message.startsWith("Parser ")) {
140+
malformed = true;
141+
} else {
142+
throw cause;
143+
}
144+
}
145+
};
146+
return {
147+
write: (chunk: string) => consume(chunk),
148+
finish: (): unknown => {
149+
consume(none);
150+
if (malformed || !complete) return undefined;
151+
selectToken(none);
152+
return assembler.done ? assembler.current : undefined;
153+
},
154+
};
155+
}

‎apps/server/src/project/AgentSessionScanner.test.ts‎

Lines changed: 18 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1522,7 +1522,7 @@ it.layer(NodeServices.layer)("AgentSessionScanner", (it) => {
15221522
}),
15231523
);
15241524

1525-
it.effect("shares a 64 MiB full-read budget across providers without hiding projects", () =>
1525+
it.effect("streams large transcripts across providers without hiding projects", () =>
15261526
Effect.gen(function* () {
15271527
const path = yield* Path.Path;
15281528
const fileSystem = yield* FileSystem.FileSystem;
@@ -1619,9 +1619,9 @@ it.layer(NodeServices.layer)("AgentSessionScanner", (it) => {
16191619
"Importable",
16201620
"Importable",
16211621
"Importable",
1622-
"Skipped",
1622+
"Importable",
16231623
]);
1624-
expect(fullReadBytes).toBe(64 * 1024 * 1024);
1624+
expect(fullReadBytes).toBe(80 * 1024 * 1024);
16251625
}),
16261626
);
16271627

@@ -1835,15 +1835,15 @@ it.layer(NodeServices.layer)("AgentSessionScanner", (it) => {
18351835
}),
18361836
);
18371837

1838-
it.effect("reports an eligible transcript over 16 MiB as skipped", () =>
1838+
it.effect("imports visible history from a transcript with an oversized tool record", () =>
18391839
Effect.gen(function* () {
18401840
const path = yield* Path.Path;
18411841
const nowMs = Date.parse("2026-08-24T12:00:00.000Z");
18421842
yield* TestClock.setTime(nowMs);
18431843
const claudeHomePath = yield* makeTempDir("t3code-claude-home-");
18441844
const codexHomePath = yield* makeTempDir("t3code-codex-home-");
18451845
const workspace = yield* makeTempDir("t3code-workspace-");
1846-
const transcript = [
1846+
const transcript = `${[
18471847
encodeTranscriptRecord({
18481848
type: "session_meta",
18491849
payload: { id: "large-session", cwd: workspace },
@@ -1852,9 +1852,10 @@ it.layer(NodeServices.layer)("AgentSessionScanner", (it) => {
18521852
type: "event_msg",
18531853
payload: { type: "user_message", message: "Import this large session" },
18541854
}),
1855-
]
1856-
.join("\n")
1857-
.padEnd(16 * 1024 * 1024 + 1, " ");
1855+
].join("\n")}\n${encodeTranscriptRecord({ type: "tool_result", data: "" }).padEnd(
1856+
16 * 1024 * 1024 + 1,
1857+
" ",
1858+
)}`;
18581859
yield* writeTranscript({
18591860
filePath: path.join(codexHomePath, "sessions", "2026", "08", "24", "rollout-large.jsonl"),
18601861
contents: transcript,
@@ -1867,7 +1868,15 @@ it.layer(NodeServices.layer)("AgentSessionScanner", (it) => {
18671868
workspaceRoot: workspace,
18681869
});
18691870

1870-
expect(outcomes).toEqual([{ _tag: "Skipped" }]);
1871+
expect(outcomes).toMatchObject([
1872+
{
1873+
_tag: "Importable",
1874+
thread: {
1875+
providerSessionId: "large-session",
1876+
messages: [{ role: "user", text: "Import this large session" }],
1877+
},
1878+
},
1879+
]);
18711880
}),
18721881
);
18731882

0 commit comments

Comments
 (0)