Skip to content

Commit 28663a2

Browse files
committed
fix(responses): enforce aggregate UTF-16 backlog budget for adapter event queue
1 parent 8a03072 commit 28663a2

2 files changed

Lines changed: 84 additions & 3 deletions

File tree

‎src/adapters/run-turn-queue.ts‎

Lines changed: 33 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,29 @@ export const PREFLIGHT_HEARTBEAT_RETAIN_LIMIT = 16;
66

77
/**
88
* Coalescing threshold for adjacent text/thinking deltas buffered with no
9-
* waiting reader (UTF-16 code units). This is a merge-size ceiling, not a
10-
* byte-memory cap: a single oversized incoming event stays one item.
9+
* waiting reader (UTF-16 code units). The aggregate backlog budget below is
10+
* enforced separately, including for a single oversized incoming event.
1111
*/
1212
export const COALESCE_MAX_CHUNK_LENGTH = 64 * 1024;
13+
/** Maximum retained string payload across one queue, measured as UTF-16 code units. */
14+
export const DEFAULT_MAX_BACKLOG_CODE_UNITS = 1024 * 1024;
15+
16+
function retainedStringCodeUnits(value: unknown, seen = new Set<object>()): number {
17+
if (typeof value === "string") return value.length;
18+
if (!value || typeof value !== "object" || seen.has(value)) return 0;
19+
seen.add(value);
20+
let total = 0;
21+
for (const nested of Object.values(value)) total += retainedStringCodeUnits(nested, seen);
22+
return total;
23+
}
24+
25+
function retainedEventStringCodeUnits(event: AdapterEvent): number {
26+
let total = 0;
27+
for (const [key, value] of Object.entries(event)) {
28+
if (key !== "type") total += retainedStringCodeUnits(value);
29+
}
30+
return total;
31+
}
1332

1433
export interface AdapterEventQueue {
1534
/**
@@ -73,11 +92,14 @@ export async function preflightAdapterEvents(
7392

7493
export function createAdapterEventQueue(opts?: {
7594
maxBacklog?: number;
95+
maxBacklogCodeUnits?: number;
7696
onBacklogExceeded?: () => void;
7797
}): AdapterEventQueue {
7898
const queued: AdapterEvent[] = [];
7999
const readers: QueueReader[] = [];
80100
const maxBacklog = opts?.maxBacklog ?? 1_024;
101+
const maxBacklogCodeUnits = opts?.maxBacklogCodeUnits ?? DEFAULT_MAX_BACKLOG_CODE_UNITS;
102+
let backlogCodeUnits = 0;
81103
let closed = false;
82104

83105
// Merge an incoming delta into the buffered tail when no reader is waiting.
@@ -124,6 +146,14 @@ export function createAdapterEventQueue(opts?: {
124146
reader({ done: false, value: event });
125147
return false;
126148
}
149+
const eventCodeUnits = retainedEventStringCodeUnits(event);
150+
if (eventCodeUnits > maxBacklogCodeUnits - backlogCodeUnits) {
151+
opts?.onBacklogExceeded?.();
152+
queued.push({ type: "error", message: "consumer stalled: adapter event backlog exceeded — turn aborted" });
153+
close();
154+
return false;
155+
}
156+
backlogCodeUnits += eventCodeUnits;
127157
if (coalesceIntoTail(event)) return true;
128158
if (queued.length >= maxBacklog) {
129159
opts?.onBacklogExceeded?.();
@@ -147,6 +177,7 @@ export function createAdapterEventQueue(opts?: {
147177
while (true) {
148178
const next = queued.shift();
149179
if (next) {
180+
backlogCodeUnits -= retainedEventStringCodeUnits(next);
150181
yield next;
151182
continue;
152183
}

‎tests/adapters/run-turn-queue.test.ts‎

Lines changed: 51 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import { describe, expect, test } from "bun:test";
2-
import { COALESCE_MAX_CHUNK_LENGTH, createAdapterEventQueue, PREFLIGHT_HEARTBEAT_RETAIN_LIMIT, preflightAdapterEvents } from "../../src/adapters/run-turn-queue";
2+
import { COALESCE_MAX_CHUNK_LENGTH, createAdapterEventQueue, DEFAULT_MAX_BACKLOG_CODE_UNITS, PREFLIGHT_HEARTBEAT_RETAIN_LIMIT, preflightAdapterEvents } from "../../src/adapters/run-turn-queue";
33
import type { AdapterEvent } from "../../src/types";
44

55
const text = (value: string): AdapterEvent => ({ type: "text_delta", text: value });
@@ -113,6 +113,56 @@ describe("run-turn adapter event queue", () => {
113113
expect(collected[0]).toEqual(thinking(Array.from({ length: 5_000 }, (_, i) => String(i % 10)).join("")));
114114
});
115115

116+
test("coalesced deltas cannot exceed the aggregate string backlog budget", async () => {
117+
let backlogExceeded = 0;
118+
const queue = createAdapterEventQueue({
119+
maxBacklogCodeUnits: 4,
120+
onBacklogExceeded: () => { backlogExceeded += 1; },
121+
});
122+
123+
queue.push(text("ab"));
124+
queue.push(text("cd"));
125+
queue.push(text("e"));
126+
127+
expect(backlogExceeded).toBe(1);
128+
expect(await queue.collect()).toEqual([
129+
text("abcd"),
130+
{ type: "error", message: "consumer stalled: adapter event backlog exceeded — turn aborted" },
131+
]);
132+
});
133+
134+
test("an oversized individual event is rejected before it is retained", async () => {
135+
let backlogExceeded = 0;
136+
const queue = createAdapterEventQueue({
137+
onBacklogExceeded: () => { backlogExceeded += 1; },
138+
});
139+
140+
queue.push(text("x".repeat(DEFAULT_MAX_BACKLOG_CODE_UNITS + 1)));
141+
142+
expect(backlogExceeded).toBe(1);
143+
expect(await queue.collect()).toEqual([
144+
{ type: "error", message: "consumer stalled: adapter event backlog exceeded — turn aborted" },
145+
]);
146+
});
147+
148+
test("consuming buffered events releases aggregate string budget", async () => {
149+
let backlogExceeded = 0;
150+
const queue = createAdapterEventQueue({
151+
maxBacklogCodeUnits: 4,
152+
onBacklogExceeded: () => { backlogExceeded += 1; },
153+
});
154+
const iterator = queue.stream()[Symbol.asyncIterator]();
155+
156+
queue.push(text("abcd"));
157+
expect(await iterator.next()).toEqual({ done: false, value: text("abcd") });
158+
queue.push(thinking("wxyz"));
159+
queue.close();
160+
161+
expect(await iterator.next()).toEqual({ done: false, value: thinking("wxyz") });
162+
expect(await iterator.next()).toEqual({ done: true, value: undefined });
163+
expect(backlogExceeded).toBe(0);
164+
});
165+
116166
test("coalescing splits past the combined-length threshold and preserves concatenation", async () => {
117167
const queue = createAdapterEventQueue();
118168
const chunk = "x".repeat(Math.floor(COALESCE_MAX_CHUNK_LENGTH / 3) + 1);

0 commit comments

Comments
 (0)