Skip to content

Commit 003de31

Browse files
test(server): a failed restart start leaves its inherited work open
Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1 parent e9298af commit 003de31

1 file changed

Lines changed: 232 additions & 1 deletion

File tree

‎apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts‎

Lines changed: 232 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ import {
44
EventId,
55
MessageId,
66
type ModelSelection,
7+
NodeId,
8+
type OrchestrationV2DomainEvent,
79
type OrchestrationV2ProviderCapabilities,
810
type OrchestrationV2ProviderSession,
911
type OrchestrationV2ProviderThread,
@@ -14,7 +16,9 @@ import {
1416
ProviderThreadId,
1517
ProviderTurnId,
1618
type RunId,
19+
RuntimeRequestId,
1720
ThreadId,
21+
TurnItemId,
1822
} from "@t3tools/contracts";
1923
import * as DateTime from "effect/DateTime";
2024
import * as Effect from "effect/Effect";
@@ -86,6 +90,116 @@ interface RestartAdapterState {
8690
}>;
8791
readonly closedSessionCount: number;
8892
readonly failedReplacementOpen: boolean;
93+
/** Every replacement open fails, not just the first. */
94+
readonly replacementOpenAlwaysFails?: boolean;
95+
/** The first turn leaves an approval, a streaming reply and a background command open. */
96+
readonly firstTurnLeavesWorkOpen?: boolean;
97+
}
98+
99+
// Work a turn leaves open when it is superseded: an approval it is waiting on,
100+
// a reply still streaming, and a background command.
101+
function openTurnWork(
102+
providerSessionId: ProviderSessionId,
103+
active: ActiveTurn,
104+
now: DateTime.Utc,
105+
): ReadonlyArray<ProviderAdapterV2Event> {
106+
const { input, providerTurnId } = active;
107+
const base = {
108+
threadId: input.threadId,
109+
runId: input.runId,
110+
providerThreadId: input.providerThread.id,
111+
providerTurnId,
112+
nativeItemRef: null,
113+
parentItemId: null,
114+
title: null,
115+
startedAt: now,
116+
completedAt: null,
117+
updatedAt: now,
118+
};
119+
const approvalNodeId = NodeId.make(`node:approval:${input.attemptId}`);
120+
const requestId = RuntimeRequestId.make(`request:approval:${input.attemptId}`);
121+
return [
122+
{
123+
type: "node.updated",
124+
driver,
125+
node: {
126+
id: approvalNodeId,
127+
threadId: input.threadId,
128+
runId: input.runId,
129+
parentNodeId: input.rootNodeId,
130+
rootNodeId: input.rootNodeId,
131+
kind: "approval_request",
132+
status: "waiting",
133+
countsForRun: false,
134+
providerThreadId: input.providerThread.id,
135+
providerTurnId,
136+
nativeItemRef: null,
137+
runtimeRequestId: requestId,
138+
checkpointScopeId: null,
139+
startedAt: now,
140+
completedAt: null,
141+
},
142+
},
143+
{
144+
type: "runtime_request.updated",
145+
driver,
146+
threadId: input.threadId,
147+
runtimeRequest: {
148+
id: requestId,
149+
nodeId: approvalNodeId,
150+
providerTurnId,
151+
nativeRequestRef: null,
152+
kind: "command",
153+
status: "pending",
154+
responseCapability: { type: "live", providerSessionId },
155+
createdAt: now,
156+
resolvedAt: null,
157+
},
158+
},
159+
{
160+
type: "turn_item.updated",
161+
driver,
162+
turnItem: {
163+
...base,
164+
id: TurnItemId.make(`turn-item:approval:${input.attemptId}`),
165+
nodeId: approvalNodeId,
166+
ordinal: 1,
167+
status: "waiting",
168+
type: "approval_request",
169+
requestId,
170+
requestKind: "command",
171+
},
172+
},
173+
{
174+
type: "message.updated",
175+
driver,
176+
message: {
177+
id: MessageId.make(`message:reply:${input.attemptId}`),
178+
threadId: input.threadId,
179+
runId: input.runId,
180+
nodeId: input.rootNodeId,
181+
role: "assistant",
182+
text: "Working on it",
183+
attachments: [],
184+
streaming: true,
185+
createdAt: now,
186+
updatedAt: now,
187+
},
188+
},
189+
{
190+
type: "turn_item.updated",
191+
driver,
192+
turnItem: {
193+
...base,
194+
id: TurnItemId.make(`turn-item:command:${input.attemptId}`),
195+
nodeId: input.rootNodeId,
196+
ordinal: 2,
197+
status: "running",
198+
type: "command_execution",
199+
input: "npm run dev",
200+
},
201+
},
202+
];
89203
}
90204

91205
function makeRestartAdapter(
@@ -108,7 +222,7 @@ function makeRestartAdapter(
108222
const failThisOpen = yield* Ref.modify(state, (current) => {
109223
const shouldFail =
110224
sessionInput.modelSelection.model === replacementSelection.model &&
111-
!current.failedReplacementOpen;
225+
(current.replacementOpenAlwaysFails === true || !current.failedReplacementOpen);
112226
return [
113227
shouldFail,
114228
{
@@ -258,6 +372,15 @@ function makeRestartAdapter(
258372
completedAt: null,
259373
},
260374
});
375+
if ((yield* Ref.get(state)).firstTurnLeavesWorkOpen === true) {
376+
for (const event of openTurnWork(
377+
sessionInput.providerSessionId,
378+
active,
379+
occurredAt,
380+
)) {
381+
yield* Queue.offer(events, event);
382+
}
383+
}
261384
return;
262385
}
263386
yield* publishTerminal(active, "completed");
@@ -537,6 +660,114 @@ it.live("restarts selection as a new attempt and retries after old-session clean
537660
),
538661
);
539662

663+
// A restart only replaces the root turn: the request, reply and command the
664+
// superseded attempt left open become the restarted run's to settle. When the
665+
// replacement never starts, failing the run must end them too, or the thread
666+
// waits on work nothing is running.
667+
it.live("settles the work a restarted run inherited when its replacement never opens", () =>
668+
Effect.scoped(
669+
Effect.gen(function* () {
670+
const name = "selection-restart-failed-start";
671+
const cwd = yield* checkpointWorkspace(name);
672+
const threadId = ThreadId.make(`thread:${name}`);
673+
const state = yield* Ref.make<RestartAdapterState>({
674+
activeTurn: null,
675+
opened: [],
676+
started: [],
677+
closedSessionCount: 0,
678+
failedReplacementOpen: false,
679+
replacementOpenAlwaysFails: true,
680+
firstTurnLeavesWorkOpen: true,
681+
});
682+
const registry = ProviderAdapterRegistry.makeSingleLayer(makeRestartAdapter(state));
683+
684+
const projection = yield* Effect.gen(function* () {
685+
const orchestrator = yield* Orchestrator.OrchestratorV2;
686+
const awaitDomainEvent = (matches: (event: OrchestrationV2DomainEvent) => boolean) =>
687+
orchestrator.streamDomainEvents.pipe(
688+
Stream.filter(matches),
689+
Stream.take(1),
690+
Stream.runDrain,
691+
Effect.forkScoped({ startImmediately: true }),
692+
);
693+
yield* orchestrator.dispatch({
694+
type: "thread.create",
695+
createdBy: "user",
696+
creationSource: "web",
697+
commandId: CommandId.make(`${name}:create`),
698+
threadId,
699+
projectId: ProjectId.make(`project:${name}`),
700+
title: name,
701+
modelSelection: initialSelection,
702+
runtimeMode: "full-access",
703+
interactionMode: "default",
704+
branch: null,
705+
worktreePath: cwd,
706+
});
707+
// The command item is the last open work the first turn reports.
708+
const workOpen = yield* awaitDomainEvent(
709+
(event) =>
710+
event.type === "turn-item.updated" && event.payload.type === "command_execution",
711+
);
712+
yield* orchestrator.dispatch({
713+
type: "message.dispatch",
714+
createdBy: "user",
715+
creationSource: "web",
716+
commandId: CommandId.make(`${name}:first`),
717+
threadId,
718+
messageId: MessageId.make(`${name}:first`),
719+
text: "first",
720+
attachments: [],
721+
modelSelection: initialSelection,
722+
dispatchMode: { type: "start_immediately" },
723+
});
724+
yield* Fiber.join(workOpen);
725+
const runId = (yield* orchestrator.getThreadProjection(threadId)).runs[0]?.id;
726+
if (runId === undefined) return yield* Effect.die("the first run is missing");
727+
728+
const runFailed = yield* awaitDomainEvent(
729+
(event) => event.type === "run.updated" && event.payload.status === "failed",
730+
);
731+
yield* orchestrator.dispatch({
732+
type: "message.dispatch",
733+
createdBy: "user",
734+
creationSource: "web",
735+
commandId: CommandId.make(`${name}:second`),
736+
threadId,
737+
messageId: MessageId.make(`${name}:second`),
738+
text: "second",
739+
attachments: [],
740+
modelSelection: replacementSelection,
741+
dispatchMode: { type: "restart_active", targetRunId: runId },
742+
});
743+
yield* Fiber.join(runFailed);
744+
return yield* orchestrator.getThreadProjection(threadId);
745+
}).pipe(Effect.provide(makeOrchestratorV2ReplayLayerWithRegistry({ name }, registry)));
746+
747+
assert.deepEqual(
748+
projection.runs.map((run) => run.status),
749+
["failed"],
750+
);
751+
assert.deepEqual(
752+
projection.runtimeRequests.map((request) => request.status),
753+
["cancelled"],
754+
);
755+
assert.isFalse(projection.messages.some((message) => message.streaming));
756+
assert.deepEqual(
757+
projection.turnItems.flatMap((item) =>
758+
item.type === "approval_request" || item.type === "command_execution"
759+
? [[item.type, item.status]]
760+
: [],
761+
),
762+
[
763+
["approval_request", "failed"],
764+
["command_execution", "failed"],
765+
],
766+
);
767+
}),
768+
),
769+
);
770+
540771
for (const deadStatus of ["stopped", "error"] as const) {
541772
it.live(
542773
`restarts the live session on a model change when a newer ${deadStatus} session record exists`,

0 commit comments

Comments
 (0)