Skip to content

Commit e27de4e

Browse files
fix(server): delegated completions preserve finished parent runs (pingdotgg#20025)
Co-authored-by: Julius Marminge <[email protected]>
1 parent 55ba1ba commit e27de4e

3 files changed

Lines changed: 178 additions & 6 deletions

File tree

‎apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts‎

Lines changed: 142 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
import * as SourceControlProviderRegistry from "../sourceControl/SourceControlProviderRegistry.ts";
22
import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderSessions";
33
import * as NodeServices from "@effect/platform-node/NodeServices";
4-
import { assert, it } from "@effect/vitest";
4+
import { assert, it, vi } from "@effect/vitest";
55
import {
6+
CheckpointId,
67
CommandId,
78
EventId,
89
MessageId,
@@ -1070,6 +1071,146 @@ const seedRestartCancelledChild = (input: {
10701071
it.layer(Layer.merge(layerTest, ProviderContinuationRequests.layer))(
10711072
"subagent follow-ups",
10721073
(it) => {
1074+
it.effect("keeps a checkpoint completion committed between finalize's read and write", () =>
1075+
Effect.gen(function* () {
1076+
const orchestrator = yield* Orchestrator.OrchestratorV2;
1077+
const sink = yield* EventSink.EventSinkV2;
1078+
const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests;
1079+
const now = yield* DateTime.now;
1080+
const threadId = ThreadId.make("thread:completion-capture-race");
1081+
const projectId = ProjectId.make("project:completion-capture-race");
1082+
const runId = RunId.make("run:completion-capture-race");
1083+
const rootNodeId = NodeId.make("node:completion-capture-race");
1084+
const checkpointId = CheckpointId.make("checkpoint:completion-capture-race");
1085+
yield* seedParentWithTerminalTask({
1086+
threadId,
1087+
projectId,
1088+
runId,
1089+
rootNodeId,
1090+
taskId: NodeId.make("node:completion-capture-seed"),
1091+
deliveryState: "acknowledged",
1092+
now,
1093+
});
1094+
const parent = (yield* orchestrator.getThreadProjection(threadId)).runs[0]!;
1095+
const waiting = { ...parent, status: "waiting" as const };
1096+
yield* sink.write({
1097+
events: [
1098+
{
1099+
id: EventId.make("event:completion-capture-waiting"),
1100+
type: "run.updated",
1101+
threadId,
1102+
runId,
1103+
occurredAt: now,
1104+
payload: waiting,
1105+
},
1106+
],
1107+
});
1108+
const child = yield* seedRestartCancelledChild({
1109+
parentThreadId: threadId,
1110+
projectId,
1111+
parentRunId: runId,
1112+
rootNodeId,
1113+
name: "completion-capture-child",
1114+
completionWake: "always",
1115+
continuationPending: false,
1116+
runStatus: "completed",
1117+
now,
1118+
});
1119+
const write = sink.writeWithEffects;
1120+
let captured = false;
1121+
const committedParentRuns: Array<OrchestrationV2Run> = [];
1122+
yield* Effect.acquireUseRelease(
1123+
Effect.sync(() =>
1124+
vi.spyOn(sink, "writeWithEffects").mockImplementation((input) =>
1125+
Effect.gen(function* () {
1126+
if (
1127+
!captured &&
1128+
input.events.some(
1129+
(event) =>
1130+
event.type === "run.updated" &&
1131+
event.payload.id === runId &&
1132+
event.payload.delegatedCompletion?.delivery != null,
1133+
)
1134+
) {
1135+
captured = true;
1136+
// Commit capture after finalize planned its stale waiting snapshot.
1137+
const { delegatedCompletion: _cohort, ...captureRun } = waiting;
1138+
yield* sink.commitCommand({
1139+
commandId: CommandId.make("command:completion-capture-race"),
1140+
threadId,
1141+
commandType: "checkpoint.capture",
1142+
acceptedAt: now,
1143+
effects: [],
1144+
events: [
1145+
{
1146+
id: EventId.make("event:completion-capture-completed"),
1147+
type: "run.updated",
1148+
threadId,
1149+
runId,
1150+
occurredAt: now,
1151+
payload: {
1152+
...captureRun,
1153+
status: "completed",
1154+
completedAt: now,
1155+
checkpointId,
1156+
},
1157+
},
1158+
],
1159+
});
1160+
}
1161+
const stored = yield* write(input);
1162+
for (const { event } of stored) {
1163+
if (event.type === "run.updated" && event.payload.id === runId) {
1164+
committedParentRuns.push(event.payload);
1165+
}
1166+
}
1167+
return stored;
1168+
}),
1169+
),
1170+
),
1171+
() => orchestrator.recoverDelegatedTasks,
1172+
(spy) => Effect.sync(() => spy.mockRestore()),
1173+
);
1174+
assert.isTrue(captured);
1175+
assert.equal(committedParentRuns.length, 1);
1176+
const committedParentRun = committedParentRuns[0];
1177+
assert.equal(committedParentRun?.status, "completed");
1178+
assert.deepEqual(committedParentRun?.completedAt, now);
1179+
assert.equal(committedParentRun?.checkpointId, checkpointId);
1180+
assert.deepEqual(committedParentRun?.delegatedCompletion?.delivery?.taskIds, [
1181+
child.taskId,
1182+
]);
1183+
const finished = yield* orchestrator.getThreadProjection(threadId);
1184+
const completed = finished.runs.find((run) => run.id === runId)!;
1185+
assert.equal(completed.status, "completed");
1186+
assert.deepEqual(completed.completedAt, now);
1187+
assert.equal(completed.checkpointId, checkpointId);
1188+
assert.deepEqual(completed.delegatedCompletion?.delivery?.taskIds, [child.taskId]);
1189+
const wake = yield* requests.take;
1190+
assert.equal(wake.delegatedCompletion?.parentRunId, runId);
1191+
const delivery = completed.delegatedCompletion!.delivery!;
1192+
yield* orchestrator.dispatch({
1193+
type: "message.dispatch",
1194+
createdBy: "agent",
1195+
creationSource: "server",
1196+
commandId: CommandId.make("command:completion-capture-delivery"),
1197+
threadId,
1198+
messageId: delivery.messageId,
1199+
text: "Child finished",
1200+
attachments: [],
1201+
delegatedCompletion: {
1202+
parentRunId: runId,
1203+
generation: delivery.generation,
1204+
taskIds: delivery.taskIds,
1205+
},
1206+
dispatchMode: { type: "queue_after_active" },
1207+
});
1208+
const delivered = yield* orchestrator.getThreadProjection(threadId);
1209+
const deliveryRun = delivered.runs.find((run) => run.userMessageId === delivery.messageId);
1210+
assert.isDefined(deliveryRun);
1211+
assert.notEqual(deliveryRun?.status, "queued");
1212+
}),
1213+
);
10731214
it.effect("pauses queued follow-up cards and resumes them", () =>
10741215
Effect.gen(function* () {
10751216
const orchestrator = yield* Orchestrator.OrchestratorV2;

‎apps/server/src/orchestration-v2/EventSink.ts‎

Lines changed: 31 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,11 @@ export interface EventSinkV2Shape {
7777
readonly events: ReadonlyArray<OrchestrationV2DomainEvent>;
7878
}) => Effect.Effect<ReadonlyArray<OrchestrationV2StoredEvent>, EventSinkV2Error>;
7979
readonly writeWithEffects: (input: {
80+
/**
81+
* Rebuilds each run.updated from the stored row with only its delegatedCompletion replaced.
82+
* The batch must not also change that run's lifecycle.
83+
*/
84+
readonly updateDelegatedCompletionOnly?: boolean;
8085
readonly unlessNativeThreadOwned?: Pick<
8186
OrchestrationV2ProviderThread,
8287
"driver" | "providerInstanceId" | "appThreadId" | "nativeThreadRef"
@@ -395,10 +400,34 @@ const layerBase: Layer.Layer<
395400
`;
396401
if (owners.length > 0) return [];
397402
}
403+
// Checkpoint capture can finish after a cohort writer reads its parent run.
404+
// Merge in this transaction so the event log and projection keep that completion.
405+
const events = input.updateDelegatedCompletionOnly
406+
? yield* Effect.forEach(input.events, (event) =>
407+
Effect.gen(function* () {
408+
if (event.type !== "run.updated") return event;
409+
const { runs } = yield* projectionStore.getThreadRecords(
410+
event.threadId,
411+
["runs"],
412+
{ runIds: [event.payload.id] },
413+
);
414+
const current = runs[0];
415+
return current === undefined
416+
? event
417+
: {
418+
...event,
419+
payload: {
420+
...current,
421+
delegatedCompletion: event.payload.delegatedCompletion,
422+
},
423+
};
424+
}),
425+
)
426+
: input.events;
398427
const normalized = yield* normalizeEvents(
399428
input.guardPendingUserInputCancellations === true
400-
? yield* guardUserInputCancellations(input.events)
401-
: input.events,
429+
? yield* guardUserInputCancellations(events)
430+
: events,
402431
);
403432
const committed = yield* eventStore.append({
404433
...(input.commandId === undefined ? {} : { commandId: input.commandId }),

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1047,11 +1047,12 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
10471047
Effect.fnUntraced(function* (
10481048
events: ReadonlyArray<Omit<OrchestrationV2DomainEvent, "id">>,
10491049
effects: ReadonlyArray<PendingOrchestrationEffectV2> = [],
1050+
options: { readonly updateDelegatedCompletionOnly?: boolean } = {},
10501051
) {
10511052
const withIds = yield* Effect.forEach(events, (event) =>
10521053
makeSystemEvent(event as Omit<OrchestrationV2DomainEvent, "id">),
10531054
);
1054-
yield* eventSink.writeWithEffects({ events: withIds, effects });
1055+
yield* eventSink.writeWithEffects({ events: withIds, effects, ...options });
10551056
});
10561057

10571058
const completionDeliveryRun = (
@@ -10551,7 +10552,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1055110552
consumedAt: now,
1055210553
};
1055310554

10554-
yield* writeSystemEvents([
10555+
const completionEvents: ReadonlyArray<Omit<OrchestrationV2DomainEvent, "id">> = [
1055510556
{
1055610557
type: "subagent.updated",
1055710558
threadId: parentThreadId,
@@ -10652,7 +10653,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1065210653
occurredAt: now,
1065310654
payload: resultTransfer,
1065410655
},
10655-
]);
10656+
];
10657+
yield* writeSystemEvents(completionEvents, [], { updateDelegatedCompletionOnly: true });
1065610658

1065710659
if (completionPlan.offer && completionPlan.parentRun !== undefined) {
1065810660
yield* offerDelegatedCompletionDelivery(parentThreadId, completionPlan.parentRun.id);

0 commit comments

Comments
 (0)