Skip to content

Commit bbce6c8

Browse files
committed
fix(server): delegated completions preserve finished parent runs
1 parent c079503 commit bbce6c8

3 files changed

Lines changed: 178 additions & 6 deletions

File tree

‎apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts‎

Lines changed: 142 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
import * as SourceControlProviderRegistry from "../sourceControl/SourceControlProviderRegistry.ts";
22
import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderSessions";
33
import * as NodeServices from "@effect/platform-node/NodeServices";
4-
import { assert, it } from "@effect/vitest";
4+
import { assert, it, vi } from "@effect/vitest";
55
import {
6+
CheckpointId,
67
CommandId,
78
EventId,
89
MessageId,
@@ -935,6 +936,146 @@ const seedRestartCancelledChild = (input: {
935936
it.layer(Layer.merge(layerTest, ProviderContinuationRequests.layer))(
936937
"subagent follow-ups",
937938
(it) => {
939+
it.effect("keeps a checkpoint completion committed between finalize's read and write", () =>
940+
Effect.gen(function* () {
941+
const orchestrator = yield* Orchestrator.OrchestratorV2;
942+
const sink = yield* EventSink.EventSinkV2;
943+
const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests;
944+
const now = yield* DateTime.now;
945+
const threadId = ThreadId.make("thread:completion-capture-race");
946+
const projectId = ProjectId.make("project:completion-capture-race");
947+
const runId = RunId.make("run:completion-capture-race");
948+
const rootNodeId = NodeId.make("node:completion-capture-race");
949+
const checkpointId = CheckpointId.make("checkpoint:completion-capture-race");
950+
yield* seedParentWithTerminalTask({
951+
threadId,
952+
projectId,
953+
runId,
954+
rootNodeId,
955+
taskId: NodeId.make("node:completion-capture-seed"),
956+
deliveryState: "acknowledged",
957+
now,
958+
});
959+
const parent = (yield* orchestrator.getThreadProjection(threadId)).runs[0]!;
960+
const waiting = { ...parent, status: "waiting" as const };
961+
yield* sink.write({
962+
events: [
963+
{
964+
id: EventId.make("event:completion-capture-waiting"),
965+
type: "run.updated",
966+
threadId,
967+
runId,
968+
occurredAt: now,
969+
payload: waiting,
970+
},
971+
],
972+
});
973+
const child = yield* seedRestartCancelledChild({
974+
parentThreadId: threadId,
975+
projectId,
976+
parentRunId: runId,
977+
rootNodeId,
978+
name: "completion-capture-child",
979+
completionWake: "always",
980+
continuationPending: false,
981+
runStatus: "completed",
982+
now,
983+
});
984+
const write = sink.writeWithEffects;
985+
let captured = false;
986+
const committedParentRuns: Array<OrchestrationV2Run> = [];
987+
yield* Effect.acquireUseRelease(
988+
Effect.sync(() =>
989+
vi.spyOn(sink, "writeWithEffects").mockImplementation((input) =>
990+
Effect.gen(function* () {
991+
if (
992+
!captured &&
993+
input.events.some(
994+
(event) =>
995+
event.type === "run.updated" &&
996+
event.payload.id === runId &&
997+
event.payload.delegatedCompletion?.delivery != null,
998+
)
999+
) {
1000+
captured = true;
1001+
// Commit capture after finalize planned its stale waiting snapshot.
1002+
const { delegatedCompletion: _cohort, ...captureRun } = waiting;
1003+
yield* sink.commitCommand({
1004+
commandId: CommandId.make("command:completion-capture-race"),
1005+
threadId,
1006+
commandType: "checkpoint.capture",
1007+
acceptedAt: now,
1008+
effects: [],
1009+
events: [
1010+
{
1011+
id: EventId.make("event:completion-capture-completed"),
1012+
type: "run.updated",
1013+
threadId,
1014+
runId,
1015+
occurredAt: now,
1016+
payload: {
1017+
...captureRun,
1018+
status: "completed",
1019+
completedAt: now,
1020+
checkpointId,
1021+
},
1022+
},
1023+
],
1024+
});
1025+
}
1026+
const stored = yield* write(input);
1027+
for (const { event } of stored) {
1028+
if (event.type === "run.updated" && event.payload.id === runId) {
1029+
committedParentRuns.push(event.payload);
1030+
}
1031+
}
1032+
return stored;
1033+
}),
1034+
),
1035+
),
1036+
() => orchestrator.recoverDelegatedTasks,
1037+
(spy) => Effect.sync(() => spy.mockRestore()),
1038+
);
1039+
assert.isTrue(captured);
1040+
assert.equal(committedParentRuns.length, 1);
1041+
const committedParentRun = committedParentRuns[0];
1042+
assert.equal(committedParentRun?.status, "completed");
1043+
assert.deepEqual(committedParentRun?.completedAt, now);
1044+
assert.equal(committedParentRun?.checkpointId, checkpointId);
1045+
assert.deepEqual(committedParentRun?.delegatedCompletion?.delivery?.taskIds, [
1046+
child.taskId,
1047+
]);
1048+
const finished = yield* orchestrator.getThreadProjection(threadId);
1049+
const completed = finished.runs.find((run) => run.id === runId)!;
1050+
assert.equal(completed.status, "completed");
1051+
assert.deepEqual(completed.completedAt, now);
1052+
assert.equal(completed.checkpointId, checkpointId);
1053+
assert.deepEqual(completed.delegatedCompletion?.delivery?.taskIds, [child.taskId]);
1054+
const wake = yield* requests.take;
1055+
assert.equal(wake.delegatedCompletion?.parentRunId, runId);
1056+
const delivery = completed.delegatedCompletion!.delivery!;
1057+
yield* orchestrator.dispatch({
1058+
type: "message.dispatch",
1059+
createdBy: "agent",
1060+
creationSource: "server",
1061+
commandId: CommandId.make("command:completion-capture-delivery"),
1062+
threadId,
1063+
messageId: delivery.messageId,
1064+
text: "Child finished",
1065+
attachments: [],
1066+
delegatedCompletion: {
1067+
parentRunId: runId,
1068+
generation: delivery.generation,
1069+
taskIds: delivery.taskIds,
1070+
},
1071+
dispatchMode: { type: "queue_after_active" },
1072+
});
1073+
const delivered = yield* orchestrator.getThreadProjection(threadId);
1074+
const deliveryRun = delivered.runs.find((run) => run.userMessageId === delivery.messageId);
1075+
assert.isDefined(deliveryRun);
1076+
assert.notEqual(deliveryRun?.status, "queued");
1077+
}),
1078+
);
9381079
const outcomes = [
9391080
"completed",
9401081
"failed",

‎apps/server/src/orchestration-v2/EventSink.ts‎

Lines changed: 31 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,11 @@ export interface EventSinkV2Shape {
7878
readonly events: ReadonlyArray<OrchestrationV2DomainEvent>;
7979
}) => Effect.Effect<ReadonlyArray<OrchestrationV2StoredEvent>, EventSinkV2Error>;
8080
readonly writeWithEffects: (input: {
81+
/**
82+
* Rebuilds each run.updated from the stored row with only its delegatedCompletion replaced.
83+
* The batch must not also change that run's lifecycle.
84+
*/
85+
readonly updateDelegatedCompletionOnly?: boolean;
8186
readonly guardPendingUserInputCancellations?: boolean;
8287
readonly commandId?: CommandId;
8388
readonly events: ReadonlyArray<OrchestrationV2DomainEvent>;
@@ -387,10 +392,34 @@ const layerBase: Layer.Layer<
387392

388393
return yield* commitThenPublish(
389394
Effect.gen(function* () {
395+
// Checkpoint capture can finish after a cohort writer reads its parent run.
396+
// Merge in this transaction so the event log and projection keep that completion.
397+
const events = input.updateDelegatedCompletionOnly
398+
? yield* Effect.forEach(input.events, (event) =>
399+
Effect.gen(function* () {
400+
if (event.type !== "run.updated") return event;
401+
const { runs } = yield* projectionStore.getThreadRecords(
402+
event.threadId,
403+
["runs"],
404+
{ runIds: [event.payload.id] },
405+
);
406+
const current = runs[0];
407+
return current === undefined
408+
? event
409+
: {
410+
...event,
411+
payload: {
412+
...current,
413+
delegatedCompletion: event.payload.delegatedCompletion,
414+
},
415+
};
416+
}),
417+
)
418+
: input.events;
390419
const normalized = yield* normalizeEvents(
391420
input.guardPendingUserInputCancellations === true
392-
? yield* guardUserInputCancellations(input.events)
393-
: input.events,
421+
? yield* guardUserInputCancellations(events)
422+
: events,
394423
);
395424
const committed = yield* eventStore.append({
396425
...(input.commandId === undefined ? {} : { commandId: input.commandId }),

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1037,12 +1037,13 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
10371037
const writeSystemEvents = (
10381038
events: ReadonlyArray<Omit<OrchestrationV2DomainEvent, "id">>,
10391039
effects: ReadonlyArray<PendingOrchestrationEffectV2> = [],
1040+
options: { readonly updateDelegatedCompletionOnly?: boolean } = {},
10401041
) =>
10411042
Effect.gen(function* () {
10421043
const withIds = yield* Effect.forEach(events, (event) =>
10431044
makeSystemEvent(event as Omit<OrchestrationV2DomainEvent, "id">),
10441045
);
1045-
yield* eventSink.writeWithEffects({ events: withIds, effects });
1046+
yield* eventSink.writeWithEffects({ events: withIds, effects, ...options });
10461047
});
10471048

10481049
const completionDeliveryRun = (
@@ -10321,7 +10322,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1032110322
consumedAt: now,
1032210323
};
1032310324

10324-
yield* writeSystemEvents([
10325+
const completionEvents: ReadonlyArray<Omit<OrchestrationV2DomainEvent, "id">> = [
1032510326
{
1032610327
type: "subagent.updated",
1032710328
threadId: parentThreadId,
@@ -10422,7 +10423,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1042210423
occurredAt: now,
1042310424
payload: resultTransfer,
1042410425
},
10425-
]);
10426+
];
10427+
yield* writeSystemEvents(completionEvents, [], { updateDelegatedCompletionOnly: true });
1042610428

1042710429
if (completionPlan.offer && completionPlan.parentRun !== undefined) {
1042810430
yield* offerDelegatedCompletionDelivery(parentThreadId, completionPlan.parentRun.id);

0 commit comments

Comments
 (0)