Skip to content

Commit 7e71853

Browse files
committed
fix(server): withdraw approvals when provider turns fail
1 parent 9e85538 commit 7e71853

7 files changed

Lines changed: 231 additions & 98 deletions

File tree

‎apps/server/src/orchestration-v2/EventSink.ts‎

Lines changed: 14 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ export interface EventSinkV2Shape {
7373
OrchestrationV2ProviderThread,
7474
"driver" | "providerInstanceId" | "appThreadId" | "nativeThreadRef"
7575
>;
76-
readonly guardPendingUserInputCancellations?: boolean;
76+
readonly guardPendingRuntimeRequestCancellations?: boolean;
7777
readonly commandId?: CommandId;
7878
readonly events: ReadonlyArray<OrchestrationV2DomainEvent>;
7979
}) => Effect.Effect<ReadonlyArray<OrchestrationV2StoredEvent>, EventSinkV2Error>;
@@ -87,13 +87,13 @@ export interface EventSinkV2Shape {
8787
OrchestrationV2ProviderThread,
8888
"driver" | "providerInstanceId" | "appThreadId" | "nativeThreadRef"
8989
>;
90-
readonly guardPendingUserInputCancellations?: boolean;
90+
readonly guardPendingRuntimeRequestCancellations?: boolean;
9191
readonly commandId?: CommandId;
9292
readonly events: ReadonlyArray<OrchestrationV2DomainEvent>;
9393
readonly effects: ReadonlyArray<EffectOutbox.PendingOrchestrationEffectV2>;
9494
}) => Effect.Effect<ReadonlyArray<OrchestrationV2StoredEvent>, EventSinkV2Error>;
9595
readonly writeIfRunCurrent: (input: {
96-
readonly guardPendingUserInputCancellations?: boolean;
96+
readonly guardPendingRuntimeRequestCancellations?: boolean;
9797
readonly commandId?: CommandId;
9898
readonly threadId: ThreadId;
9999
readonly runId: RunId;
@@ -110,7 +110,7 @@ export interface EventSinkV2Shape {
110110
>;
111111
/** Commits only if the expected attempt and ordinal still own the provider thread, protecting newer attempts. */
112112
readonly writeIfProviderThreadOwner: (input: {
113-
readonly guardPendingUserInputCancellations?: boolean;
113+
readonly guardPendingRuntimeRequestCancellations?: boolean;
114114
readonly commandId?: CommandId;
115115
readonly providerThreadId: ProviderThreadId;
116116
readonly runId: RunId;
@@ -285,25 +285,20 @@ const layerBase: Layer.Layer<
285285

286286
// A user can answer after terminal normalization reads the pending request.
287287
// Recheck inside the write transaction so stale cleanup cannot erase answers.
288-
const guardUserInputCancellations =
288+
const guardRuntimeRequestCancellations =
289289
/* This helper runs for individual orchestration events; the enclosing operation owns the span. */
290290
Effect.fnUntraced(function* (events: ReadonlyArray<OrchestrationV2DomainEvent>) {
291291
const staleRequests = new Set<RuntimeRequestId>();
292292
const staleNodes = new Set<NodeId>();
293293
for (const event of events) {
294-
if (
295-
event.type !== "runtime-request.updated" ||
296-
event.payload.kind !== "user_input" ||
297-
event.payload.status !== "cancelled"
298-
)
294+
if (event.type !== "runtime-request.updated" || event.payload.status !== "cancelled")
299295
continue;
300296
const current = yield* projectionStore.getRuntimeRequest(
301297
event.threadId,
302298
event.payload.id,
303299
);
304300
if (
305301
current?.status !== "pending" ||
306-
current.kind !== "user_input" ||
307302
current.providerTurnId !== event.payload.providerTurnId ||
308303
current.responseCapability.type === "message"
309304
) {
@@ -319,7 +314,8 @@ const layerBase: Layer.Layer<
319314
return event.payload.status !== "cancelled" || !staleNodes.has(event.payload.id);
320315
case "turn-item.updated":
321316
return (
322-
event.payload.type !== "user_input_request" ||
317+
(event.payload.type !== "user_input_request" &&
318+
event.payload.type !== "approval_request") ||
323319
event.payload.status !== "cancelled" ||
324320
!staleRequests.has(event.payload.requestId)
325321
);
@@ -438,8 +434,8 @@ const layerBase: Layer.Layer<
438434
? yield* updateDelegatedCompletionOnly(input.events, () => true)
439435
: input.events;
440436
const normalized = yield* normalizeEvents(
441-
input.guardPendingUserInputCancellations === true
442-
? yield* guardUserInputCancellations(events)
437+
input.guardPendingRuntimeRequestCancellations === true
438+
? yield* guardRuntimeRequestCancellations(events)
443439
: events,
444440
);
445441
const committed = yield* eventStore.append({
@@ -497,8 +493,8 @@ const layerBase: Layer.Layer<
497493
}
498494

499495
const normalized = yield* normalizeEvents(
500-
input.guardPendingUserInputCancellations === true
501-
? yield* guardUserInputCancellations(input.events)
496+
input.guardPendingRuntimeRequestCancellations === true
497+
? yield* guardRuntimeRequestCancellations(input.events)
502498
: input.events,
503499
);
504500
const storedEvents = yield* eventStore.append({
@@ -562,8 +558,8 @@ const layerBase: Layer.Layer<
562558
}
563559

564560
const normalized = yield* normalizeEvents(
565-
input.guardPendingUserInputCancellations === true
566-
? yield* guardUserInputCancellations(input.events)
561+
input.guardPendingRuntimeRequestCancellations === true
562+
? yield* guardRuntimeRequestCancellations(input.events)
567563
: input.events,
568564
);
569565
const storedEvents = yield* eventStore.append({

‎apps/server/src/orchestration-v2/ProjectionStore.ts‎

Lines changed: 38 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -315,7 +315,7 @@ export type ProjectionRuntimeRecoveryState = Pick<
315315
| "turnItems"
316316
>;
317317

318-
export type ProjectionPendingUserInputs = Pick<
318+
export type ProjectionPendingRuntimeRequests = Pick<
319319
OrchestrationV2ThreadProjection,
320320
"runtimeRequests" | "nodes" | "turnItems"
321321
>;
@@ -606,10 +606,11 @@ export interface ProjectionStoreV2Shape {
606606
readonly getRuntimeRecoveryProjection: (
607607
threadId: ThreadId,
608608
) => Effect.Effect<ProjectionRuntimeRecoveryState, ProjectionStoreV2Error>;
609-
readonly getPendingNativeUserInputs: (
609+
readonly getPendingNativeRuntimeRequests: (
610610
threadId: ThreadId,
611611
providerTurnId: ProviderTurnId,
612-
) => Effect.Effect<ProjectionPendingUserInputs, ProjectionStoreV2Error>;
612+
includeApprovals?: boolean,
613+
) => Effect.Effect<ProjectionPendingRuntimeRequests, ProjectionStoreV2Error>;
613614
readonly getPlan: (
614615
threadId: ThreadId,
615616
planId: PlanId,
@@ -4927,46 +4928,51 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
49274928
? cause
49284929
: new ProjectionStoreReadError({ threadId, cause });
49294930

4930-
const getPendingNativeUserInputs: ProjectionStoreV2Shape["getPendingNativeUserInputs"] = (
4931-
threadId,
4932-
providerTurnId,
4933-
) =>
4934-
sql
4935-
.withTransaction(
4936-
Effect.gen(function* () {
4937-
const rows = yield* sql<PayloadRow>`
4931+
const getPendingNativeRuntimeRequests: ProjectionStoreV2Shape["getPendingNativeRuntimeRequests"] =
4932+
(threadId, providerTurnId, includeApprovals = false) =>
4933+
sql
4934+
.withTransaction(
4935+
Effect.gen(function* () {
4936+
const rows = yield* sql<PayloadRow>`
49384937
SELECT payload_json FROM orchestration_v2_projection_runtime_requests
49394938
WHERE thread_id = ${threadId} AND provider_turn_id = ${providerTurnId}
4940-
AND kind = 'user_input' AND status = 'pending'
4939+
AND status = 'pending'
49414940
`;
4942-
const runtimeRequests = (yield* decodeRows(
4943-
decodeRuntimeRequestPayload,
4944-
threadId,
4945-
)(rows)).filter((request) => request.responseCapability.type !== "message");
4946-
if (runtimeRequests.length === 0) return { runtimeRequests, nodes: [], turnItems: [] };
4947-
const nodeRows = yield* sql<PayloadRow>`
4941+
const runtimeRequests = (yield* decodeRows(
4942+
decodeRuntimeRequestPayload,
4943+
threadId,
4944+
)(rows)).filter(
4945+
(request) =>
4946+
request.responseCapability.type !== "message" &&
4947+
(includeApprovals || request.kind === "user_input"),
4948+
);
4949+
if (runtimeRequests.length === 0)
4950+
return { runtimeRequests, nodes: [], turnItems: [] };
4951+
const nodeRows = yield* sql<PayloadRow>`
49484952
SELECT payload_json FROM orchestration_v2_projection_nodes
49494953
WHERE thread_id = ${threadId} AND ${sql.in(
49504954
"node_id",
49514955
runtimeRequests.map((request) => request.nodeId),
49524956
)}
49534957
`;
4954-
const itemRows = yield* sql<PayloadRow>`
4958+
const itemRows = yield* sql<PayloadRow>`
49554959
SELECT payload_json FROM orchestration_v2_projection_turn_items
49564960
WHERE thread_id = ${threadId} AND provider_turn_id = ${providerTurnId}
4957-
AND type = 'user_input_request'
4961+
AND type IN ('user_input_request', 'approval_request')
49584962
`;
4959-
const requestIds = new Set(runtimeRequests.map((request) => request.id));
4960-
return {
4961-
runtimeRequests,
4962-
nodes: yield* decodeRows(decodeNodePayload, threadId)(nodeRows),
4963-
turnItems: (yield* decodeRows(decodeTurnItemPayload, threadId)(itemRows)).filter(
4964-
(item) => item.type === "user_input_request" && requestIds.has(item.requestId),
4965-
),
4966-
};
4967-
}),
4968-
)
4969-
.pipe(Effect.mapError(controlReadError(threadId)));
4963+
const requestIds = new Set(runtimeRequests.map((request) => request.id));
4964+
return {
4965+
runtimeRequests,
4966+
nodes: yield* decodeRows(decodeNodePayload, threadId)(nodeRows),
4967+
turnItems: (yield* decodeRows(decodeTurnItemPayload, threadId)(itemRows)).filter(
4968+
(item) =>
4969+
(item.type === "user_input_request" || item.type === "approval_request") &&
4970+
requestIds.has(item.requestId),
4971+
),
4972+
};
4973+
}),
4974+
)
4975+
.pipe(Effect.mapError(controlReadError(threadId)));
49704976

49714977
const getPlan: ProjectionStoreV2Shape["getPlan"] = (threadId, planId) =>
49724978
sql
@@ -7401,7 +7407,7 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
74017407
getRunMessage: readOnReader(getRunMessage),
74027408
canStartQueuedRun: readOnReader(canStartQueuedRun),
74037409
getHeldQueuedRuns: readOnReader(getHeldQueuedRuns),
7404-
getPendingNativeUserInputs: readOnReader(getPendingNativeUserInputs),
7410+
getPendingNativeRuntimeRequests: readOnReader(getPendingNativeRuntimeRequests),
74057411
hasUnpairedRunInterruptRequest: readOnReader(hasUnpairedRunInterruptRequest),
74067412
isNodeStored: readOnReader(isNodeStored),
74077413
getMessageCount: readOnReader(getMessageCount),

‎apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts‎

Lines changed: 23 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1111,8 +1111,8 @@ layer("ProviderEventIngestorV2", (it) => {
11111111
}),
11121112
);
11131113

1114-
it.effect.each(["interrupted", "control"] as const)(
1115-
"dismisses only native questions when a provider turn ends with %s",
1114+
it.effect.each(["interrupted", "failed", "completed", "cancelled", "control"] as const)(
1115+
"withdraws approvals only when explicitly abandoned (%s)",
11161116
(terminal) =>
11171117
Effect.gen(function* () {
11181118
const now = yield* DateTime.now;
@@ -1227,6 +1227,7 @@ layer("ProviderEventIngestorV2", (it) => {
12271227
providerSessionId,
12281228
providerInstanceId: modelSelection.instanceId,
12291229
threadId,
1230+
withdrawApprovals: terminal === "control",
12301231
event:
12311232
terminal === "control"
12321233
? {
@@ -1258,7 +1259,10 @@ layer("ProviderEventIngestorV2", (it) => {
12581259
const stored = yield* ingestor.ingestNormalized(input);
12591260
const projection = yield* projectionStore.getThreadProjection(threadId);
12601261
for (const fixture of fixtures) {
1261-
const closed = fixture.key === "native" || fixture.key === "unavailable";
1262+
const closed =
1263+
fixture.key === "native" ||
1264+
fixture.key === "unavailable" ||
1265+
(fixture.key === "approval" && terminal === "control");
12621266
const request = projection.runtimeRequests.find(
12631267
(item) => item.id === fixture.request.id,
12641268
)!;
@@ -1279,10 +1283,10 @@ layer("ProviderEventIngestorV2", (it) => {
12791283
}
12801284
assert.equal(
12811285
stored.filter((entry) => entry.event.type === "runtime-request.updated").length,
1282-
2,
1286+
terminal === "control" ? 3 : 2,
12831287
);
12841288
assert.isEmpty(
1285-
(yield* projectionStore.getPendingNativeUserInputs(threadId, providerTurnId))
1289+
(yield* projectionStore.getPendingNativeRuntimeRequests(threadId, providerTurnId))
12861290
.runtimeRequests,
12871291
);
12881292
const replayed = yield* eventStore.read({ threadId }).pipe(Stream.runCollect);
@@ -1292,16 +1296,16 @@ layer("ProviderEventIngestorV2", (it) => {
12921296
entry.event.type === "runtime-request.updated" &&
12931297
entry.event.payload.status === "cancelled",
12941298
).length,
1295-
2,
1299+
terminal === "control" ? 3 : 2,
12961300
);
12971301
const repeated = yield* ingestor.ingestNormalized(input);
12981302
assert.isFalse(repeated.some((entry) => entry.event.type === "runtime-request.updated"));
12991303
}),
13001304
);
13011305

1302-
it.effect(
1303-
"preserves an answer committed after terminal normalization reads a pending question",
1304-
() =>
1306+
it.effect.each(["question", "approval"] as const)(
1307+
"preserves an answer committed after terminal normalization reads a pending %s",
1308+
(kind) =>
13051309
Effect.scoped(
13061310
Effect.gen(function* () {
13071311
const eventSink = yield* EventSink.EventSinkV2;
@@ -1329,7 +1333,7 @@ layer("ProviderEventIngestorV2", (it) => {
13291333
nodeId,
13301334
providerTurnId,
13311335
nativeRequestRef: null,
1332-
kind: "user_input",
1336+
kind: kind === "approval" ? "command" : "user_input",
13331337
status: "pending",
13341338
responseCapability: { type: "live", providerSessionId },
13351339
createdAt: now,
@@ -1341,7 +1345,7 @@ layer("ProviderEventIngestorV2", (it) => {
13411345
runId: null,
13421346
parentNodeId: null,
13431347
rootNodeId: nodeId,
1344-
kind: "user_input_request",
1348+
kind: kind === "approval" ? "approval_request" : "user_input_request",
13451349
status: "waiting",
13461350
countsForRun: false,
13471351
providerThreadId,
@@ -1367,9 +1371,9 @@ layer("ProviderEventIngestorV2", (it) => {
13671371
startedAt: now,
13681372
completedAt: null,
13691373
updatedAt: now,
1370-
type: "user_input_request",
1371-
requestId: request.id,
1372-
questions: [],
1374+
...(kind === "approval"
1375+
? { type: "approval_request", requestId: request.id, requestKind: "command" }
1376+
: { type: "user_input_request", requestId: request.id, questions: [] }),
13731377
};
13741378
const seedEvents: Array<OrchestrationV2DomainEvent> = [threadEvent];
13751379
for (const payload of [
@@ -1400,6 +1404,7 @@ layer("ProviderEventIngestorV2", (it) => {
14001404
);
14011405
const terminal = yield* ingestor
14021406
.ingestNormalized({
1407+
withdrawApprovals: kind === "approval",
14031408
providerSessionId,
14041409
providerInstanceId: modelSelection.instanceId,
14051410
threadId,
@@ -1422,11 +1427,12 @@ layer("ProviderEventIngestorV2", (it) => {
14221427
.pipe(Effect.forkScoped);
14231428
yield* Deferred.await(normalized);
14241429
const answers = { decision: "Use the existing workspace" };
1430+
const response = kind === "approval" ? { decision: "accept" as const } : { answers };
14251431
const responseEvents: Array<OrchestrationV2DomainEvent> = [];
14261432
for (const payload of [
14271433
{
14281434
type: "runtime-request.updated" as const,
1429-
payload: { ...request, status: "resolved" as const, answers, resolvedAt: now },
1435+
payload: { ...request, status: "resolved" as const, ...response, resolvedAt: now },
14301436
},
14311437
{
14321438
type: "node.updated" as const,
@@ -1453,7 +1459,8 @@ layer("ProviderEventIngestorV2", (it) => {
14531459
const projection = yield* projections.getThreadProjection(threadId);
14541460
const answered = projection.runtimeRequests.find((entry) => entry.id === request.id)!;
14551461
assert.equal(answered.status, "resolved");
1456-
assert.deepEqual(answered.answers, answers);
1462+
if (kind === "approval") assert.equal(answered.decision, "accept");
1463+
else assert.deepEqual(answered.answers, answers);
14571464
assert.equal(projection.nodes.find((entry) => entry.id === nodeId)!.status, "completed");
14581465
assert.equal(
14591466
projection.turnItems.find((entry) => entry.id === item.id)!.status,

0 commit comments

Comments
 (0)