Skip to content

Commit f4e863f

Browse files
committed
fix(server): snapshot windows keep the limited run that notification wakes hide
A snapshot window reads only the newest rows, so an older usage-limited run, its root error and the notification starts of the wakes after it could be missing. Clients then could not find the limited run to resume it. Read that evidence after the window and add it to runs and turnItems, never to the visible rows.
1 parent 0f21292 commit f4e863f

2 files changed

Lines changed: 207 additions & 1 deletion

File tree

‎apps/server/src/orchestration-v2/ProjectionStore.test.ts‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import { assert, it, vi } from "@effect/vitest";
2+
import { usageLimitBlockedRun } from "@t3tools/shared/orchestrationV2ThreadError";
23
import {
34
EnvironmentId,
45
EventId,
@@ -6355,6 +6356,117 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => {
63556356
}),
63566357
);
63576358

6359+
it.effect("snapshot windows keep the limited run that notification wakes hide", () =>
6360+
Effect.gen(function* () {
6361+
const store = yield* ProjectionStore.ProjectionStoreV2;
6362+
const threadId = yield* addRolledBackRecoveryCandidate("limit-window");
6363+
const original = (yield* store.getThreadProjection(threadId)).runs[0]!;
6364+
const now = yield* DateTime.now;
6365+
const base = { threadId, providerThreadId: null, providerTurnId: null, nativeItemRef: null };
6366+
yield* store.apply({
6367+
id: EventId.make("event:limit-window:error"),
6368+
type: "turn-item.updated",
6369+
threadId,
6370+
occurredAt: now,
6371+
payload: {
6372+
...base,
6373+
id: TurnItemId.make("limit-window:error"),
6374+
runId: original.id,
6375+
nodeId: original.rootNodeId,
6376+
parentItemId: null,
6377+
ordinal: 2,
6378+
status: "failed",
6379+
title: "Usage limit reached",
6380+
startedAt: now,
6381+
completedAt: now,
6382+
updatedAt: now,
6383+
type: "error",
6384+
failure: {
6385+
class: "usage_limit",
6386+
message: "Plan limit reached.",
6387+
resetAt: "2099-01-01T00:00:00.000Z",
6388+
code: "usageLimitExceeded",
6389+
retryable: null,
6390+
},
6391+
},
6392+
});
6393+
yield* store.apply({
6394+
id: EventId.make("event:limit-window:failed"),
6395+
type: "run.updated",
6396+
threadId,
6397+
occurredAt: now,
6398+
payload: { ...original, status: "failed", completedAt: now },
6399+
});
6400+
for (let index = 0; index < 6; index++) {
6401+
const runId = RunId.make(`run:limit-window:wake-${index}`);
6402+
const nodeId = NodeId.make(`node:limit-window:wake-${index}`);
6403+
const at = DateTime.add(now, { seconds: index + 1 });
6404+
yield* store.apply({
6405+
id: EventId.make(`event:limit-window:wake-${index}`),
6406+
type: "run.created",
6407+
threadId,
6408+
runId,
6409+
nodeId,
6410+
driver,
6411+
providerInstanceId,
6412+
occurredAt: at,
6413+
payload: {
6414+
...original,
6415+
id: runId,
6416+
ordinal: original.ordinal + index + 1,
6417+
rootNodeId: nodeId,
6418+
userMessageId: MessageId.make(`message:limit-window:wake-${index}`),
6419+
status: "completed",
6420+
startedAt: at,
6421+
completedAt: at,
6422+
},
6423+
});
6424+
yield* store.apply({
6425+
id: EventId.make(`event:limit-window:wake-${index}:notification`),
6426+
type: "turn-item.updated",
6427+
threadId,
6428+
runId,
6429+
nodeId,
6430+
driver,
6431+
occurredAt: at,
6432+
payload: {
6433+
...base,
6434+
id: TurnItemId.make(`limit-window:wake-${index}`),
6435+
runId,
6436+
nodeId,
6437+
parentItemId: null,
6438+
ordinal: 10 + index,
6439+
status: "completed",
6440+
title: null,
6441+
startedAt: at,
6442+
completedAt: at,
6443+
updatedAt: at,
6444+
type: "notification",
6445+
source: { kind: "delegated_task", taskIds: [] },
6446+
outcome: "failed",
6447+
summary: "Subagent failed",
6448+
},
6449+
});
6450+
}
6451+
6452+
const snapshot = yield* store.getThreadSnapshotWindow(threadId, { rowLimit: 3 });
6453+
const bounded = buildBoundedThreadProjection({
6454+
projection: snapshot.projection,
6455+
snapshotSequence: snapshot.snapshotSequence,
6456+
}).projection;
6457+
for (const projection of [snapshot.projection, bounded]) {
6458+
assert.equal(
6459+
usageLimitBlockedRun(projection.runs, projection.turnItems, null)?.id,
6460+
original.id,
6461+
);
6462+
}
6463+
assert.notInclude(
6464+
snapshot.projection.visibleTurnItems.map((row) => String(row.sourceItemId)),
6465+
"limit-window:error",
6466+
);
6467+
}),
6468+
);
6469+
63586470
it.effect("selects the latest waiting secret only from active runs", () =>
63596471
Effect.gen(function* () {
63606472
const projectionStore = yield* ProjectionStore.ProjectionStoreV2;

‎apps/server/src/orchestration-v2/ProjectionStore.ts‎

Lines changed: 95 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1071,6 +1071,7 @@ const encodeTimelineSources = Schema.encodeEffect(
10711071
Schema.Array(Schema.Struct({ threadId: Schema.String, id: Schema.String })),
10721072
),
10731073
);
1074+
const LIMIT_EVIDENCE_MAX_RUNS = 200;
10741075
const encodeIdList = Schema.encodeSync(Schema.fromJsonString(Schema.Array(Schema.String)));
10751076

10761077
// Empty filters still scan SQLite rows; short-circuit them, including turnItemTypes: [].
@@ -6066,6 +6067,99 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
60666067
),
60676068
);
60686069

6070+
// A window reads only the newest rows, so an older usage-limited run and the notification
6071+
// starts of the wakes after it can be missing. Clients need them (usageLimitBlockedRun)
6072+
// to resume it. They join runs and turnItems, never visibleTurnItems.
6073+
const withLimitEvidence = Effect.fn("ProjectionStore.withLimitEvidence")(function* (
6074+
threadId: ThreadId,
6075+
projection: OrchestrationV2ThreadProjection,
6076+
) {
6077+
const rows = yield* sql<{ readonly kind: string; readonly id: string }>`
6078+
WITH blocked AS (
6079+
SELECT r.run_id, r.ordinal, item.turn_item_id AS error_id
6080+
FROM orchestration_v2_projection_threads t
6081+
${latestFailureJoins}
6082+
INNER JOIN orchestration_v2_projection_runs r ON r.run_id = (
6083+
SELECT latest.run_id FROM orchestration_v2_projection_runs latest
6084+
WHERE latest.thread_id = t.thread_id
6085+
AND latest.status <> 'queued'
6086+
AND NOT (
6087+
latest.status = 'cancelled'
6088+
AND json_extract(latest.payload_json, '$.startedAt') IS NULL
6089+
)
6090+
AND NOT ${isLimitedWake("latest")}
6091+
ORDER BY latest.completed_at IS NULL DESC, latest.completed_at DESC,
6092+
latest.ordinal DESC, latest.run_id DESC
6093+
LIMIT 1
6094+
) AND r.status = 'failed'
6095+
INNER JOIN orchestration_v2_projection_turn_items item ON item.turn_item_id = (
6096+
SELECT error.turn_item_id FROM orchestration_v2_projection_turn_items error
6097+
WHERE error.thread_id = t.thread_id AND error.run_id = r.run_id
6098+
AND error.type = 'error' AND error.status = 'failed'
6099+
AND error.node_id IS json_extract(r.payload_json, '$.rootNodeId')
6100+
ORDER BY error.updated_at DESC, error.ordinal DESC, error.turn_item_id DESC
6101+
LIMIT 1
6102+
)
6103+
WHERE t.thread_id = ${threadId}
6104+
AND EXISTS (
6105+
SELECT 1 FROM orchestration_v2_projection_turn_items limited
6106+
WHERE limited.thread_id = t.thread_id
6107+
AND limited.type = 'error' AND limited.status = 'failed'
6108+
AND json_extract(limited.payload_json, '$.failure.class') = 'usage_limit'
6109+
)
6110+
AND json_extract(item.payload_json, '$.failure.class') = 'usage_limit'
6111+
), later AS (
6112+
-- Runs omitted here are not loaded at all, which leaves the result unchanged.
6113+
SELECT run.run_id, run.payload_json
6114+
FROM orchestration_v2_projection_runs run, blocked
6115+
WHERE run.thread_id = ${threadId} AND run.ordinal > blocked.ordinal
6116+
ORDER BY run.ordinal DESC
6117+
LIMIT ${LIMIT_EVIDENCE_MAX_RUNS}
6118+
)
6119+
SELECT 'run' AS kind, run_id AS id FROM blocked
6120+
UNION ALL SELECT 'item', error_id FROM blocked
6121+
UNION ALL SELECT 'run', run_id FROM later
6122+
UNION ALL
6123+
SELECT 'item', start.turn_item_id
6124+
FROM later
6125+
CROSS JOIN orchestration_v2_projection_turn_items start ON start.turn_item_id = (
6126+
SELECT first_input.turn_item_id FROM orchestration_v2_projection_turn_items first_input
6127+
WHERE first_input.run_id = later.run_id
6128+
AND first_input.node_id IS json_extract(later.payload_json, '$.rootNodeId')
6129+
AND first_input.type IN ('user_message', 'notification')
6130+
ORDER BY first_input.ordinal
6131+
LIMIT 1
6132+
)
6133+
WHERE start.type = 'notification'
6134+
`;
6135+
const loadedRunIds = new Set<string>(projection.runs.map((run) => run.id));
6136+
const loadedItemIds = new Set<string>(projection.turnItems.map((item) => item.id));
6137+
const runIds = rows.flatMap((row) =>
6138+
row.kind === "run" && !loadedRunIds.has(row.id) ? [row.id] : [],
6139+
);
6140+
const itemIds = rows.flatMap((row) =>
6141+
row.kind === "item" && !loadedItemIds.has(row.id) ? [row.id] : [],
6142+
);
6143+
if (runIds.length === 0 && itemIds.length === 0) return projection;
6144+
const runRows = yield* sql<PayloadRow>`
6145+
SELECT payload_json FROM orchestration_v2_projection_runs
6146+
WHERE thread_id = ${threadId}
6147+
AND run_id IN (SELECT value FROM json_each(${encodeIdList(runIds)}))
6148+
`;
6149+
const itemRows = yield* sql<PayloadRow>`
6150+
SELECT payload_json FROM orchestration_v2_projection_turn_items
6151+
WHERE thread_id = ${threadId}
6152+
AND turn_item_id IN (SELECT value FROM json_each(${encodeIdList(itemIds)}))
6153+
`;
6154+
const runs = yield* decodeRows(decodeRunPayload, threadId)(runRows);
6155+
const turnItems = yield* decodeRows(decodeTurnItemPayload, threadId)(itemRows);
6156+
return {
6157+
...projection,
6158+
runs: [...projection.runs, ...runs].toSorted((left, right) => left.ordinal - right.ordinal),
6159+
turnItems: [...projection.turnItems, ...turnItems],
6160+
};
6161+
});
6162+
60696163
const getThreadSnapshotWindow: ProjectionStoreV2Shape["getThreadSnapshotWindow"] = (
60706164
threadId,
60716165
options,
@@ -6107,7 +6201,7 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
61076201
return {
61086202
schemaVersion: ORCHESTRATION_V2_PROJECTION_SCHEMA_VERSION,
61096203
snapshotSequence: rows[0]?.snapshot_sequence ?? 0,
6110-
projection,
6204+
projection: yield* withLimitEvidence(threadId, projection),
61116205
};
61126206
}),
61136207
)

0 commit comments

Comments
 (0)