Skip to content

Commit 189b46d

Browse files
committed
fix(server): bounded thread snapshots keep the evidence for a limited run
A limited run hidden behind notification wakes is found from its root error item and from the first input of each later run. A long chain of wakes pushed those out of the newest turns in a bounded snapshot, leaving the thread unresumable on clients. Retain them like interrupt requests and count them in the dependency reserve.
1 parent 0664044 commit 189b46d

2 files changed

Lines changed: 109 additions & 2 deletions

File tree

‎apps/server/src/orchestration-v2/threadHistoryPaging.test.ts‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import {
88
type OrchestrationV2TurnItem,
99
} from "@t3tools/contracts";
1010
import { describe, expect, it } from "@effect/vitest";
11+
import { usageLimitBlockedRun } from "@t3tools/shared/orchestrationV2ThreadError";
1112
import * as DateTime from "effect/DateTime";
1213

1314
import {
@@ -676,6 +677,73 @@ describe("threadHistoryPaging", () => {
676677
expect(bounded.hasMoreHistory).toBe(true);
677678
});
678679

680+
it("keeps a limited run resumable when later notification wakes outgrow the window", () => {
681+
const limitedNode = NodeId.make("node-limited");
682+
const limited = {
683+
id: RunId.make("run-limited"),
684+
ordinal: 1,
685+
status: "failed",
686+
startedAt: NOW,
687+
completedAt: NOW,
688+
rootNodeId: limitedNode,
689+
};
690+
const limitError = {
691+
...makeRow(0).item,
692+
id: TurnItemId.make("limit-error"),
693+
type: "error",
694+
runId: limited.id,
695+
nodeId: limitedNode,
696+
status: "failed",
697+
failure: {
698+
class: "usage_limit",
699+
message: "Plan limit reached.",
700+
code: "usageLimitExceeded",
701+
retryable: null,
702+
resetAt: "2026-06-20T05:00:00.000Z",
703+
},
704+
} as unknown as OrchestrationV2TurnItem;
705+
const wakes = Array.from({ length: 6 }, (_, index) => {
706+
const nodeId = NodeId.make(`node-wake-${index}`);
707+
const run = {
708+
...limited,
709+
id: RunId.make(`run-wake-${index}`),
710+
ordinal: index + 2,
711+
status: "completed",
712+
completedAt: DateTime.add(NOW, { seconds: index + 1 }),
713+
rootNodeId: nodeId,
714+
};
715+
const row = makeRow(index + 1);
716+
const item = {
717+
...row.item,
718+
id: TurnItemId.make(`wake-notification-${index}`),
719+
type: "notification",
720+
runId: run.id,
721+
nodeId,
722+
source: { kind: "delegated_task", taskIds: [] },
723+
outcome: "failed",
724+
summary: "Subagent failed",
725+
} as unknown as OrchestrationV2TurnItem;
726+
return { run, row: { ...row, sourceItemId: item.id, item } };
727+
});
728+
const rows = [{ ...makeRow(0), sourceItemId: limitError.id, item: limitError }].concat(
729+
wakes.map(({ row }) => row),
730+
);
731+
const full = {
732+
...makeProjection(rows),
733+
runs: [limited, ...wakes.map(({ run }) => run)],
734+
} as unknown as ReturnType<typeof makeProjection>;
735+
const bounded = buildBoundedThreadProjection({
736+
projection: full,
737+
snapshotSequence: 1,
738+
policy: { maxItems: 2, maxEncodedBytes: 10_000_000 },
739+
});
740+
741+
expect(bounded.projection.visibleTurnItems).toHaveLength(2);
742+
expect(
743+
usageLimitBlockedRun(bounded.projection.runs, bounded.projection.turnItems, null)?.id,
744+
).toBe(limited.id);
745+
});
746+
679747
it("charges local row.item duplication when measuring bounded timeline bytes", () => {
680748
const rows = Array.from({ length: 8 }, (_, index) => makeRow(index, { outputBytes: 2_000 }));
681749
const full = makeProjection(rows);

‎apps/server/src/orchestration-v2/threadHistoryPaging.ts‎

Lines changed: 41 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import type {
55
ThreadId,
66
TurnItemId,
77
} from "@t3tools/contracts";
8+
import { usageLimitBlockedRun } from "@t3tools/shared/orchestrationV2ThreadError";
89

910
/**
1011
* Every window stops at the byte budget. A window that holds a user turn also
@@ -524,10 +525,46 @@ function isRetainedDependency(
524525
);
525526
}
526527

528+
// Clients find a limited run that later notification wakes hide (usageLimitBlockedRun) from its
529+
// root error and from how each later run started. A long chain of wakes pushes those out of the
530+
// newest turns, which would leave the thread unresumable.
531+
function limitRecoveryItemIds(
532+
projection: Pick<OrchestrationV2ThreadProjection, "runs" | "turnItems">,
533+
): ReadonlySet<string> {
534+
const ids = new Set<string>();
535+
const blocked = usageLimitBlockedRun(projection.runs, projection.turnItems, null);
536+
if (blocked === null) return ids;
537+
const laterRoots = new Set<string>();
538+
for (const run of projection.runs) {
539+
if (run.ordinal > blocked.ordinal && run.rootNodeId !== null) {
540+
laterRoots.add(String(run.rootNodeId));
541+
}
542+
}
543+
if (laterRoots.size === 0) return ids;
544+
const firstInputs = new Map<string, OrchestrationV2TurnItem>();
545+
for (const item of projection.turnItems) {
546+
if (item.type === "error" && item.runId === blocked.id && item.nodeId === blocked.rootNodeId) {
547+
ids.add(String(item.id));
548+
} else if (
549+
(item.type === "user_message" || item.type === "notification") &&
550+
item.nodeId !== null &&
551+
laterRoots.has(String(item.nodeId))
552+
) {
553+
const first = firstInputs.get(String(item.nodeId));
554+
if (first === undefined || item.ordinal < first.ordinal) {
555+
firstInputs.set(String(item.nodeId), item);
556+
}
557+
}
558+
}
559+
for (const item of firstInputs.values()) ids.add(String(item.id));
560+
return ids;
561+
}
562+
527563
/** Retain small requests outside the visible window; later result pages still need them for attempt/run reducers. */
528564
function retainedInterruptRequestTurnItems(
529565
projection: OrchestrationV2ThreadProjection,
530566
visible: ReadonlyArray<OrchestrationV2ProjectedTurnItem>,
567+
limitItemIds: ReadonlySet<string>,
531568
): OrchestrationV2TurnItem[] {
532569
const visibleLocalIds = new Set<string>();
533570
for (const row of visible) {
@@ -539,7 +576,7 @@ function retainedInterruptRequestTurnItems(
539576
const pendingRequestIds = pendingRequestIdsOf(projection);
540577
const retained: OrchestrationV2TurnItem[] = [];
541578
for (const item of projection.turnItems) {
542-
if (!isRetainedDependency(item, pendingRequestIds)) {
579+
if (!isRetainedDependency(item, pendingRequestIds) && !limitItemIds.has(String(item.id))) {
543580
continue;
544581
}
545582
if (visibleLocalIds.has(String(item.id))) {
@@ -657,11 +694,12 @@ export function buildBoundedThreadProjection(input: {
657694
};
658695
const latestLocalTurnOrdinal = computeLatestLocalTurnOrdinal(input.projection.turnItems);
659696

697+
const limitItemIds = limitRecoveryItemIds(controlProjection);
660698
const dependencyReserve = (() => {
661699
let reserve = 0;
662700
const pendingRequestIds = pendingRequestIdsOf(controlProjection);
663701
for (const item of controlProjection.turnItems) {
664-
if (isRetainedDependency(item, pendingRequestIds)) {
702+
if (isRetainedDependency(item, pendingRequestIds) || limitItemIds.has(String(item.id))) {
665703
reserve += bytesOfJson(item);
666704
}
667705
}
@@ -692,6 +730,7 @@ export function buildBoundedThreadProjection(input: {
692730
const dependencyTurnItems = retainedInterruptRequestTurnItems(
693731
controlProjection,
694732
visibleTurnItems,
733+
limitItemIds,
695734
);
696735
const turnItemById = new Map<string, OrchestrationV2TurnItem>();
697736
for (const item of windowTurnItems) {

0 commit comments

Comments
 (0)