Skip to content

Commit d8ce455

Browse files
committed
fix(server): delegated completions preserve finished parent runs
1 parent 5307c23 commit d8ce455

3 files changed

Lines changed: 178 additions & 6 deletions

File tree

‎apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts‎

Lines changed: 142 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
import * as SourceControlProviderRegistry from "../sourceControl/SourceControlProviderRegistry.ts";
22
import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderSessions";
33
import * as NodeServices from "@effect/platform-node/NodeServices";
4-
import { assert, it } from "@effect/vitest";
4+
import { assert, it, vi } from "@effect/vitest";
55
import {
6+
CheckpointId,
67
CommandId,
78
EventId,
89
MessageId,
@@ -924,6 +925,146 @@ const seedRestartCancelledChild = (input: {
924925
it.layer(Layer.merge(layerTest, ProviderContinuationRequests.layer))(
925926
"subagent follow-ups",
926927
(it) => {
928+
it.effect("keeps a checkpoint completion committed between finalize's read and write", () =>
929+
Effect.gen(function* () {
930+
const orchestrator = yield* Orchestrator.OrchestratorV2;
931+
const sink = yield* EventSink.EventSinkV2;
932+
const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests;
933+
const now = yield* DateTime.now;
934+
const threadId = ThreadId.make("thread:completion-capture-race");
935+
const projectId = ProjectId.make("project:completion-capture-race");
936+
const runId = RunId.make("run:completion-capture-race");
937+
const rootNodeId = NodeId.make("node:completion-capture-race");
938+
const checkpointId = CheckpointId.make("checkpoint:completion-capture-race");
939+
yield* seedParentWithTerminalTask({
940+
threadId,
941+
projectId,
942+
runId,
943+
rootNodeId,
944+
taskId: NodeId.make("node:completion-capture-seed"),
945+
deliveryState: "acknowledged",
946+
now,
947+
});
948+
const parent = (yield* orchestrator.getThreadProjection(threadId)).runs[0]!;
949+
const waiting = { ...parent, status: "waiting" as const };
950+
yield* sink.write({
951+
events: [
952+
{
953+
id: EventId.make("event:completion-capture-waiting"),
954+
type: "run.updated",
955+
threadId,
956+
runId,
957+
occurredAt: now,
958+
payload: waiting,
959+
},
960+
],
961+
});
962+
const child = yield* seedRestartCancelledChild({
963+
parentThreadId: threadId,
964+
projectId,
965+
parentRunId: runId,
966+
rootNodeId,
967+
name: "completion-capture-child",
968+
completionWake: "always",
969+
continuationPending: false,
970+
runStatus: "completed",
971+
now,
972+
});
973+
const write = sink.writeWithEffects;
974+
let captured = false;
975+
const committedParentRuns: Array<OrchestrationV2Run> = [];
976+
yield* Effect.acquireUseRelease(
977+
Effect.sync(() =>
978+
vi.spyOn(sink, "writeWithEffects").mockImplementation((input) =>
979+
Effect.gen(function* () {
980+
if (
981+
!captured &&
982+
input.events.some(
983+
(event) =>
984+
event.type === "run.updated" &&
985+
event.payload.id === runId &&
986+
event.payload.delegatedCompletion?.delivery != null,
987+
)
988+
) {
989+
captured = true;
990+
// Commit capture after finalize planned its stale waiting snapshot.
991+
const { delegatedCompletion: _cohort, ...captureRun } = waiting;
992+
yield* sink.commitCommand({
993+
commandId: CommandId.make("command:completion-capture-race"),
994+
threadId,
995+
commandType: "checkpoint.capture",
996+
acceptedAt: now,
997+
effects: [],
998+
events: [
999+
{
1000+
id: EventId.make("event:completion-capture-completed"),
1001+
type: "run.updated",
1002+
threadId,
1003+
runId,
1004+
occurredAt: now,
1005+
payload: {
1006+
...captureRun,
1007+
status: "completed",
1008+
completedAt: now,
1009+
checkpointId,
1010+
},
1011+
},
1012+
],
1013+
});
1014+
}
1015+
const stored = yield* write(input);
1016+
for (const { event } of stored) {
1017+
if (event.type === "run.updated" && event.payload.id === runId) {
1018+
committedParentRuns.push(event.payload);
1019+
}
1020+
}
1021+
return stored;
1022+
}),
1023+
),
1024+
),
1025+
() => orchestrator.recoverDelegatedTasks,
1026+
(spy) => Effect.sync(() => spy.mockRestore()),
1027+
);
1028+
assert.isTrue(captured);
1029+
assert.equal(committedParentRuns.length, 1);
1030+
const committedParentRun = committedParentRuns[0];
1031+
assert.equal(committedParentRun?.status, "completed");
1032+
assert.deepEqual(committedParentRun?.completedAt, now);
1033+
assert.equal(committedParentRun?.checkpointId, checkpointId);
1034+
assert.deepEqual(committedParentRun?.delegatedCompletion?.delivery?.taskIds, [
1035+
child.taskId,
1036+
]);
1037+
const finished = yield* orchestrator.getThreadProjection(threadId);
1038+
const completed = finished.runs.find((run) => run.id === runId)!;
1039+
assert.equal(completed.status, "completed");
1040+
assert.deepEqual(completed.completedAt, now);
1041+
assert.equal(completed.checkpointId, checkpointId);
1042+
assert.deepEqual(completed.delegatedCompletion?.delivery?.taskIds, [child.taskId]);
1043+
const wake = yield* requests.take;
1044+
assert.equal(wake.delegatedCompletion?.parentRunId, runId);
1045+
const delivery = completed.delegatedCompletion!.delivery!;
1046+
yield* orchestrator.dispatch({
1047+
type: "message.dispatch",
1048+
createdBy: "agent",
1049+
creationSource: "server",
1050+
commandId: CommandId.make("command:completion-capture-delivery"),
1051+
threadId,
1052+
messageId: delivery.messageId,
1053+
text: "Child finished",
1054+
attachments: [],
1055+
delegatedCompletion: {
1056+
parentRunId: runId,
1057+
generation: delivery.generation,
1058+
taskIds: delivery.taskIds,
1059+
},
1060+
dispatchMode: { type: "queue_after_active" },
1061+
});
1062+
const delivered = yield* orchestrator.getThreadProjection(threadId);
1063+
const deliveryRun = delivered.runs.find((run) => run.userMessageId === delivery.messageId);
1064+
assert.isDefined(deliveryRun);
1065+
assert.notEqual(deliveryRun?.status, "queued");
1066+
}),
1067+
);
9271068
it.effect("pauses queued follow-up cards and resumes them", () =>
9281069
Effect.gen(function* () {
9291070
const orchestrator = yield* Orchestrator.OrchestratorV2;

‎apps/server/src/orchestration-v2/EventSink.ts‎

Lines changed: 31 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,11 @@ export interface EventSinkV2Shape {
7272
readonly events: ReadonlyArray<OrchestrationV2DomainEvent>;
7373
}) => Effect.Effect<ReadonlyArray<OrchestrationV2StoredEvent>, EventSinkV2Error>;
7474
readonly writeWithEffects: (input: {
75+
/**
76+
* Rebuilds each run.updated from the stored row with only its delegatedCompletion replaced.
77+
* The batch must not also change that run's lifecycle.
78+
*/
79+
readonly updateDelegatedCompletionOnly?: boolean;
7580
readonly guardPendingUserInputCancellations?: boolean;
7681
readonly commandId?: CommandId;
7782
readonly events: ReadonlyArray<OrchestrationV2DomainEvent>;
@@ -369,10 +374,34 @@ const layerBase: Layer.Layer<
369374

370375
return yield* commitThenPublish(
371376
Effect.gen(function* () {
377+
// Checkpoint capture can finish after a cohort writer reads its parent run.
378+
// Merge in this transaction so the event log and projection keep that completion.
379+
const events = input.updateDelegatedCompletionOnly
380+
? yield* Effect.forEach(input.events, (event) =>
381+
Effect.gen(function* () {
382+
if (event.type !== "run.updated") return event;
383+
const { runs } = yield* projectionStore.getThreadRecords(
384+
event.threadId,
385+
["runs"],
386+
{ runIds: [event.payload.id] },
387+
);
388+
const current = runs[0];
389+
return current === undefined
390+
? event
391+
: {
392+
...event,
393+
payload: {
394+
...current,
395+
delegatedCompletion: event.payload.delegatedCompletion,
396+
},
397+
};
398+
}),
399+
)
400+
: input.events;
372401
const normalized = yield* normalizeEvents(
373402
input.guardPendingUserInputCancellations === true
374-
? yield* guardUserInputCancellations(input.events)
375-
: input.events,
403+
? yield* guardUserInputCancellations(events)
404+
: events,
376405
);
377406
const committed = yield* eventStore.append({
378407
...(input.commandId === undefined ? {} : { commandId: input.commandId }),

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1025,12 +1025,13 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
10251025
const writeSystemEvents = (
10261026
events: ReadonlyArray<Omit<OrchestrationV2DomainEvent, "id">>,
10271027
effects: ReadonlyArray<PendingOrchestrationEffectV2> = [],
1028+
options: { readonly updateDelegatedCompletionOnly?: boolean } = {},
10281029
) =>
10291030
Effect.gen(function* () {
10301031
const withIds = yield* Effect.forEach(events, (event) =>
10311032
makeSystemEvent(event as Omit<OrchestrationV2DomainEvent, "id">),
10321033
);
1033-
yield* eventSink.writeWithEffects({ events: withIds, effects });
1034+
yield* eventSink.writeWithEffects({ events: withIds, effects, ...options });
10341035
});
10351036

10361037
const completionDeliveryRun = (
@@ -10475,7 +10476,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1047510476
consumedAt: now,
1047610477
};
1047710478

10478-
yield* writeSystemEvents([
10479+
const completionEvents: ReadonlyArray<Omit<OrchestrationV2DomainEvent, "id">> = [
1047910480
{
1048010481
type: "subagent.updated",
1048110482
threadId: parentThreadId,
@@ -10576,7 +10577,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1057610577
occurredAt: now,
1057710578
payload: resultTransfer,
1057810579
},
10579-
]);
10580+
];
10581+
yield* writeSystemEvents(completionEvents, [], { updateDelegatedCompletionOnly: true });
1058010582

1058110583
if (completionPlan.offer && completionPlan.parentRun !== undefined) {
1058210584
yield* offerDelegatedCompletionDelivery(parentThreadId, completionPlan.parentRun.id);

0 commit comments

Comments
 (0)