Skip to content

Commit 88d3d6a

Browse files
committed
fix(server): stop reads only the turn items that can hide a limited run
Stop loaded every user message, error and notification in the thread to find a usage-limited run that notification wakes hide. It now reads the latest failed run's errors, then the inputs of runs completed before that failure resets. The SQL wake test also covers the two near misses: a wake completed after the reset and a run a user message started.
1 parent de93e06 commit 88d3d6a

2 files changed

Lines changed: 89 additions & 6 deletions

File tree

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 29 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -11202,12 +11202,35 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1120211202
const { runs, subagents } = input.projection;
1120311203
let { thread } = input.projection;
1120411204
let recovery = thread.limitRecovery;
11205-
const { turnItems } = yield* projectionStore
11206-
.getThreadRecords(thread.id, ["turnItems"], {
11207-
turnItemTypes: ["user_message", "error", "notification"],
11208-
})
11209-
.pipe(mapDispatchError(input.command));
11210-
// Stop must also disarm a limited run that completed notification wakes hide.
11205+
// Stop must also disarm a limited run that completed notification wakes hide. Only the
11206+
// latest failure's errors and the inputs of runs completed before its reset can matter.
11207+
const readItems = (filter: Parameters<typeof projectionStore.getThreadRecords>[2]) =>
11208+
projectionStore
11209+
.getThreadRecords(thread.id, ["turnItems"], filter)
11210+
.pipe(mapDispatchError(input.command));
11211+
const failed = latestExecutedRun(runs.filter((run) => run.status === "failed"));
11212+
const errors =
11213+
failed === null
11214+
? []
11215+
: (yield* readItems({ turnItemTypes: ["error"], turnItemRunIds: [failed.id] })).turnItems;
11216+
const resetAt = latestRootProviderFailure(failed, errors)?.resetAt;
11217+
const resetMs = resetAt == null ? Number.NaN : Date.parse(resetAt);
11218+
const wakeRunIds = runs
11219+
.filter(
11220+
(run) =>
11221+
run.status === "completed" &&
11222+
run.completedAt !== null &&
11223+
DateTime.toEpochMillis(run.completedAt) <= resetMs,
11224+
)
11225+
.map((run) => run.id);
11226+
const inputs =
11227+
wakeRunIds.length === 0
11228+
? []
11229+
: (yield* readItems({
11230+
turnItemTypes: ["user_message", "notification"],
11231+
turnItemRunIds: wakeRunIds,
11232+
})).turnItems;
11233+
const turnItems = [...errors, ...inputs];
1121111234
const latest = usageLimitBlockedRun(runs, turnItems, null) ?? latestExecutedRun(runs);
1121211235
if (latest?.status === "failed") {
1121311236
const failure = latestRootProviderFailure(latest, turnItems);

‎apps/server/src/orchestration-v2/ProjectionStore.test.ts‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6009,6 +6009,66 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => {
60096009
(yield* store.getShellSnapshot()).threads.find((row) => row.id === threadId)!.latestRunId,
60106010
original.id,
60116011
);
6012+
// Near misses: a wake completed after the reset, or a run a user message started, did work.
6013+
const assertLatest = Effect.fnUntraced(function* (runId: RunId) {
6014+
const projection = yield* store.getThreadProjection(threadId);
6015+
assert.equal(ProjectionStore.threadShellFromProjection(projection).latestRunId, runId);
6016+
assert.equal(
6017+
(yield* store.getShellSnapshot()).threads.find((row) => row.id === threadId)!.latestRunId,
6018+
runId,
6019+
);
6020+
});
6021+
const setWakeCompletedAt = (completedAt: DateTime.Utc) =>
6022+
store.apply({
6023+
id: EventId.make(`event:limit-shell:wake-at-${DateTime.formatIso(completedAt)}`),
6024+
type: "run.updated",
6025+
threadId,
6026+
runId: wakeRunId,
6027+
nodeId: wakeNodeId,
6028+
driver,
6029+
occurredAt: now,
6030+
payload: {
6031+
...wakeProjection.runs.find((run) => run.id === wakeRunId)!,
6032+
completedAt,
6033+
},
6034+
});
6035+
yield* setWakeCompletedAt(DateTime.makeUnsafe("2099-01-02T00:00:00.000Z"));
6036+
yield* assertLatest(wakeRunId);
6037+
yield* setWakeCompletedAt(DateTime.add(now, { minutes: 1 }));
6038+
yield* assertLatest(original.id);
6039+
yield* store.apply({
6040+
id: EventId.make("event:limit-shell:wake-user-start"),
6041+
type: "turn-item.updated",
6042+
threadId,
6043+
runId: wakeRunId,
6044+
nodeId: wakeNodeId,
6045+
driver,
6046+
occurredAt: now,
6047+
payload: {
6048+
createdBy: "user",
6049+
creationSource: "web",
6050+
id: TurnItemId.make("limit-shell:wake-user-start"),
6051+
threadId,
6052+
runId: wakeRunId,
6053+
nodeId: wakeNodeId,
6054+
providerThreadId: null,
6055+
providerTurnId: null,
6056+
nativeItemRef: null,
6057+
parentItemId: null,
6058+
ordinal: 3,
6059+
status: "completed",
6060+
title: null,
6061+
startedAt: now,
6062+
completedAt: now,
6063+
updatedAt: now,
6064+
type: "user_message",
6065+
messageId: MessageId.make("message:limit-shell:wake"),
6066+
inputIntent: "turn_start",
6067+
text: "a real request",
6068+
attachments: [],
6069+
},
6070+
});
6071+
yield* assertLatest(wakeRunId);
60126072
const sql = yield* SqlClient.SqlClient;
60136073
yield* sql`DELETE FROM orchestration_v2_projection_runs WHERE run_id = ${wakeRunId}`;
60146074
yield* sql`DELETE FROM orchestration_v2_projection_runs WHERE run_id = ${aheadRunId}`;

0 commit comments

Comments
 (0)