|
| 1 | +import * as SchemaAST from "effect/SchemaAST"; |
| 2 | +import { isMany, none, type Many } from "stream-chain/defs.js"; |
| 3 | +import { Assembler } from "stream-json/core/assembler.js"; |
| 4 | +import { filter } from "stream-json/core/filters/filter.js"; |
| 5 | +import * as StreamJson from "stream-json/core/parser.js"; |
| 6 | +import type { ParserOptions, Token } from "stream-json/core/parser.js"; |
| 7 | + |
| 8 | +type JsonPath = ReadonlyArray<string | number | null>; |
| 9 | + |
| 10 | +/** Select schema fields before assembling their values, without a second field list. */ |
| 11 | +export function createTranscriptJsonSelector(schema: { readonly ast: SchemaAST.AST }) { |
| 12 | + const ast = SchemaAST.toEncoded(schema.ast); |
| 13 | + const includes = (node: SchemaAST.AST, path: JsonPath, index: number): boolean => { |
| 14 | + if (index === path.length) return true; |
| 15 | + switch (node._tag) { |
| 16 | + case "Objects": |
| 17 | + // Records have dynamic keys. Keep their values for the decoder to validate. |
| 18 | + if (node.indexSignatures.length > 0) return true; |
| 19 | + return node.propertySignatures.some( |
| 20 | + (property) => |
| 21 | + String(property.name) === path[index] && includes(property.type, path, index + 1), |
| 22 | + ); |
| 23 | + case "Arrays": { |
| 24 | + const key = path[index]; |
| 25 | + if (typeof key !== "number") return true; |
| 26 | + const element = node.elements[key]; |
| 27 | + if (element) return includes(element, path, index + 1); |
| 28 | + return node.rest.length === 0 || node.rest.some((item) => includes(item, path, index + 1)); |
| 29 | + } |
| 30 | + case "Union": |
| 31 | + return node.types.some((type) => includes(type, path, index)); |
| 32 | + case "Suspend": |
| 33 | + return includes(node.thunk(), path, index); |
| 34 | + case "Unknown": |
| 35 | + case "Any": |
| 36 | + case "ObjectKeyword": |
| 37 | + case "Declaration": |
| 38 | + // Unstructured/custom schemas must reach the decoder intact. The |
| 39 | + // shared budget still bounds their allocations. |
| 40 | + return true; |
| 41 | + default: |
| 42 | + return false; |
| 43 | + } |
| 44 | + }; |
| 45 | + return (path: JsonPath) => includes(ast, path, 0); |
| 46 | +} |
| 47 | + |
| 48 | +export class TranscriptJsonLimitError extends Error {} |
| 49 | + |
| 50 | +/** |
| 51 | + * Project a single JSONL record without materializing unselected string values. |
| 52 | + * The caller supplies a shared allocation budget for the entire transcript. |
| 53 | + * Budget exhaustion rejects the transcript, never a message within it. |
| 54 | + */ |
| 55 | +export function createTranscriptJsonReader( |
| 56 | + reserve: (bytes: number) => void, |
| 57 | + selectPath: (path: JsonPath) => boolean, |
| 58 | +) { |
| 59 | + // The synchronous tokenizer is exported at runtime in 3.6.0, but omitted |
| 60 | + // from its bundled types. Unlike parser(), it does not wrap tokens in an |
| 61 | + // async generator; the file reader already supplies backpressure and UTF-8. |
| 62 | + const { jsonParser } = StreamJson as typeof StreamJson & { |
| 63 | + jsonParser: ( |
| 64 | + options: ParserOptions, |
| 65 | + ) => (input: string | typeof none) => Many<Token> | typeof none; |
| 66 | + }; |
| 67 | + const tokenize = jsonParser({ packValues: false }); |
| 68 | + const select = filter({ filter: selectPath, streamKeys: false }) as ( |
| 69 | + input: Token | typeof none, |
| 70 | + ) => Token | Many<Token> | typeof none; |
| 71 | + const assembler = new Assembler(); |
| 72 | + let key: string | null = null; |
| 73 | + let value = ""; |
| 74 | + let depth = 0; |
| 75 | + let complete = false; |
| 76 | + let malformed = false; |
| 77 | + |
| 78 | + const assemble = (token: Token) => { |
| 79 | + reserve( |
| 80 | + 64 + ("value" in token && typeof token.value === "string" ? token.value.length * 2 : 0), |
| 81 | + ); |
| 82 | + switch (token.name) { |
| 83 | + case "startString": |
| 84 | + case "startNumber": |
| 85 | + value = ""; |
| 86 | + break; |
| 87 | + case "stringChunk": |
| 88 | + case "numberChunk": |
| 89 | + value += token.value; |
| 90 | + break; |
| 91 | + case "endString": |
| 92 | + assembler.consume({ name: "stringValue", value }); |
| 93 | + value = ""; |
| 94 | + break; |
| 95 | + case "endNumber": |
| 96 | + assembler.consume({ name: "numberValue", value }); |
| 97 | + value = ""; |
| 98 | + break; |
| 99 | + default: |
| 100 | + assembler.consume(token); |
| 101 | + } |
| 102 | + }; |
| 103 | + const selectToken = (token: Token | typeof none) => { |
| 104 | + const selected = select(token); |
| 105 | + if (selected === none) return; |
| 106 | + if (isMany(selected)) { |
| 107 | + for (const item of selected.values) assemble(item); |
| 108 | + } else { |
| 109 | + assemble(selected); |
| 110 | + } |
| 111 | + }; |
| 112 | + const consume = (input: string | typeof none) => { |
| 113 | + if (malformed) return; |
| 114 | + try { |
| 115 | + const tokens = tokenize(input); |
| 116 | + if (tokens === none) return; |
| 117 | + for (const token of tokens.values) { |
| 118 | + if (token.name === "startObject" || token.name === "startArray") { |
| 119 | + if (++depth > 128) |
| 120 | + throw new TranscriptJsonLimitError("Transcript JSON nesting exceeds 128 levels"); |
| 121 | + } else if (token.name === "endObject" || token.name === "endArray") { |
| 122 | + if (--depth === 0) complete = true; |
| 123 | + } |
| 124 | + // Charge keys before assembling them, including unknown names. Reject |
| 125 | + // the transcript on exhaustion instead of silently shortening a key. |
| 126 | + if (token.name === "startKey") { |
| 127 | + key = ""; |
| 128 | + } else if (token.name === "stringChunk" && key !== null) { |
| 129 | + reserve(token.value.length * 2); |
| 130 | + key += token.value; |
| 131 | + } else if (token.name === "endKey") { |
| 132 | + selectToken({ name: "keyValue", value: key ?? "" }); |
| 133 | + key = null; |
| 134 | + } else { |
| 135 | + selectToken(token); |
| 136 | + } |
| 137 | + } |
| 138 | + } catch (cause) { |
| 139 | + if (cause instanceof Error && cause.message.startsWith("Parser ")) { |
| 140 | + malformed = true; |
| 141 | + } else { |
| 142 | + throw cause; |
| 143 | + } |
| 144 | + } |
| 145 | + }; |
| 146 | + return { |
| 147 | + write: (chunk: string) => consume(chunk), |
| 148 | + finish: (): unknown => { |
| 149 | + consume(none); |
| 150 | + if (malformed || !complete) return undefined; |
| 151 | + selectToken(none); |
| 152 | + return assembler.done ? assembler.current : undefined; |
| 153 | + }, |
| 154 | + }; |
| 155 | +} |
0 commit comments