Skip to content

Commit f12bb62

Browse files
fix(server): a restart that misses its turn starts as a new turn (#32899)
Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent 5a9866d commit f12bb62

2 files changed

Lines changed: 234 additions & 3 deletions

File tree

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6064,7 +6064,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
60646064
command.dispatchMode,
60656065
command.deliveryIntent,
60666066
);
6067-
if (dispatchMode.type === "steer_active") {
6067+
if (dispatchMode.type === "steer_active" || dispatchMode.type === "restart_active") {
60686068
const targetRunId = dispatchMode.targetRunId;
60696069
const target = projection.runs.find((run) => run.id === targetRunId);
60706070
const turn = latestProviderTurnForAttempt(
@@ -6076,24 +6076,36 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
60766076
if (
60776077
target !== undefined &&
60786078
(target.status === "completed" ||
6079+
(dispatchMode.type === "restart_active" && !isActiveRun(target)) ||
60796080
((target.status === "running" || target.status === "waiting") &&
60806081
turn !== undefined &&
6081-
turn.status === "completed"))
6082+
(turn.status === "completed" ||
6083+
(dispatchMode.type === "restart_active" &&
6084+
turn.status !== "pending" &&
6085+
turn.status !== "running"))))
60826086
) {
60836087
dispatchMode = { type: "start_immediately" };
60846088
}
60856089
}
60866090
// Stop leaves the run reading as running until its provider stops. A plain
60876091
// send in that window queues behind the stopped queue instead of steering
60886092
// text the agent never receives, and like any user send it releases the
6089-
// hold; an explicit steer or restart still fails below.
6093+
// hold; an explicit steer still fails below.
60906094
if (
60916095
command.deliveryIntent === "auto" &&
60926096
(dispatchMode.type === "steer_active" || dispatchMode.type === "restart_active") &&
60936097
(yield* stopReachedRun(command, command.threadId, dispatchMode.targetRunId))
60946098
) {
60956099
dispatchMode = { type: "queue_after_active" };
60966100
}
6101+
// An explicit restart that Stop reached has only its start left, so it
6102+
// queues behind the stopping run like any other start.
6103+
if (
6104+
dispatchMode.type === "restart_active" &&
6105+
(yield* stopReachedRun(command, command.threadId, dispatchMode.targetRunId))
6106+
) {
6107+
dispatchMode = { type: "start_immediately" };
6108+
}
60976109
const freshContext = command.freshContext === true;
60986110
if (
60996111
freshContext &&

‎apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts‎

Lines changed: 219 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1773,6 +1773,225 @@ it.effect("keeps a queued message queued when promoted into a run that Stop reac
17731773
),
17741774
);
17751775

1776+
// regression: #32891 a restart that Stop already reached is lost
1777+
it.effect("refuses a steer but starts a restart sent into a run that Stop reached", () =>
1778+
Effect.scoped(
1779+
Effect.gen(function* () {
1780+
const { started, steered, layer, startFirstTurn } =
1781+
yield* nextTurnSelectionHarness("restart-after-stop");
1782+
yield* Effect.gen(function* () {
1783+
const orchestrator = yield* Orchestrator.OrchestratorV2;
1784+
const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
1785+
const threadId = yield* startFirstTurn;
1786+
const first = started[0]!;
1787+
// The worker has not delivered the interrupt yet, so the run still reads as running.
1788+
yield* orchestrator.dispatch({
1789+
type: "run.interrupt",
1790+
commandId: CommandId.make("stop"),
1791+
threadId,
1792+
runId: first.runId,
1793+
});
1794+
const steerError = yield* orchestrator
1795+
.dispatch({
1796+
type: "message.dispatch",
1797+
commandId: CommandId.make("steer"),
1798+
threadId,
1799+
messageId: MessageId.make("message:steer"),
1800+
text: "steer",
1801+
attachments: [],
1802+
modelSelection: runSelection,
1803+
dispatchMode: { type: "steer_active", targetRunId: first.runId },
1804+
createdBy: "user",
1805+
creationSource: "web",
1806+
})
1807+
.pipe(Effect.flip);
1808+
assert.equal(steerError._tag, "OrchestratorDispatchError");
1809+
assert.include(String(steerError.cause), "is stopping and cannot be steered");
1810+
yield* orchestrator.dispatch({
1811+
type: "message.dispatch",
1812+
commandId: CommandId.make("restart"),
1813+
threadId,
1814+
messageId: MessageId.make("message:restart"),
1815+
text: "restart",
1816+
attachments: [],
1817+
modelSelection: runSelection,
1818+
dispatchMode: { type: "restart_active", targetRunId: first.runId },
1819+
createdBy: "user",
1820+
creationSource: "web",
1821+
});
1822+
yield* worker.drain();
1823+
1824+
assert.deepEqual(steered, []);
1825+
assert.deepEqual(
1826+
started.map((turn) => turn.message.text),
1827+
["first", "restart"],
1828+
);
1829+
}).pipe(Effect.provide(layer));
1830+
}),
1831+
),
1832+
);
1833+
1834+
// regression: #32891 a restart that arrives after the turn ended is lost
1835+
it.effect.each([
1836+
{ status: "completed", settled: true },
1837+
{ status: "completed", settled: false },
1838+
{ status: "interrupted", settled: true },
1839+
{ status: "interrupted", settled: false },
1840+
{ status: "cancelled", settled: false },
1841+
] as const)(
1842+
"starts a restart that missed the turn ($status, run settled: $settled) as a new turn",
1843+
({ status, settled: runSettled }) =>
1844+
Effect.scoped(
1845+
Effect.gen(function* () {
1846+
const { events, started, steered, layer, startFirstTurn } = yield* nextTurnSelectionHarness(
1847+
`restart-after-${status}-${runSettled ? "settled" : "unsettled"}`,
1848+
);
1849+
yield* Effect.gen(function* () {
1850+
const orchestrator = yield* Orchestrator.OrchestratorV2;
1851+
const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
1852+
const threadId = yield* startFirstTurn;
1853+
const first = started[0]!;
1854+
const turn = (yield* orchestrator.getThreadProjection(threadId)).providerTurns[0]!;
1855+
const turnEnded = yield* orchestrator.streamDomainEvents.pipe(
1856+
Stream.filter(
1857+
(event) =>
1858+
event.type === "provider-turn.updated" &&
1859+
event.payload.id === turn.id &&
1860+
event.payload.status === status,
1861+
),
1862+
Stream.take(1),
1863+
Stream.runDrain,
1864+
Effect.forkScoped,
1865+
);
1866+
yield* Queue.offer(events, {
1867+
type: "provider_turn.updated",
1868+
driver,
1869+
providerTurn: { ...turn, status, completedAt: yield* DateTime.now },
1870+
});
1871+
yield* Fiber.join(turnEnded);
1872+
const settleRun = Effect.gen(function* () {
1873+
const settled = yield* orchestrator.streamDomainEvents.pipe(
1874+
Stream.filter(
1875+
(event) =>
1876+
event.type === "run.updated" &&
1877+
event.payload.id === first.runId &&
1878+
event.payload.status !== "running",
1879+
),
1880+
Stream.take(1),
1881+
Stream.runDrain,
1882+
Effect.forkScoped,
1883+
);
1884+
yield* Queue.offer(events, {
1885+
type: "turn.terminal",
1886+
driver,
1887+
providerThreadId: turn.providerThreadId,
1888+
providerTurnId: turn.id,
1889+
runOrdinal: first.runOrdinal,
1890+
status,
1891+
failure: null,
1892+
threadDisposition: "reusable",
1893+
});
1894+
yield* Fiber.join(settled);
1895+
yield* worker.drain();
1896+
});
1897+
if (runSettled) yield* settleRun;
1898+
1899+
yield* orchestrator.dispatch({
1900+
type: "message.dispatch",
1901+
commandId: CommandId.make("restart"),
1902+
threadId,
1903+
messageId: MessageId.make("message:restart"),
1904+
text: "restart",
1905+
attachments: [],
1906+
modelSelection: runSelection,
1907+
dispatchMode: { type: "restart_active", targetRunId: first.runId },
1908+
createdBy: "user",
1909+
creationSource: "web",
1910+
});
1911+
yield* worker.drain();
1912+
if (!runSettled) {
1913+
const restartLeftQueue = yield* orchestrator.streamDomainEvents.pipe(
1914+
Stream.filter(
1915+
(event) =>
1916+
event.type === "run.updated" &&
1917+
event.payload.userMessageId === "message:restart" &&
1918+
event.payload.status !== "queued",
1919+
),
1920+
Stream.take(1),
1921+
Stream.runDrain,
1922+
Effect.forkScoped,
1923+
);
1924+
yield* settleRun;
1925+
yield* Fiber.join(restartLeftQueue);
1926+
yield* worker.drain();
1927+
}
1928+
1929+
assert.deepEqual(steered, []);
1930+
assert.deepEqual(
1931+
started.map((turn) => turn.message.text),
1932+
["first", "restart"],
1933+
);
1934+
}).pipe(Effect.provide(layer));
1935+
}),
1936+
),
1937+
);
1938+
1939+
// regression: #32891 a restart into a failed provider turn is lost
1940+
it.effect("queues a restart sent into a provider turn that failed before its run settled", () =>
1941+
Effect.scoped(
1942+
Effect.gen(function* () {
1943+
const { events, started, steered, layer, startFirstTurn } = yield* nextTurnSelectionHarness(
1944+
"restart-after-failed-unsettled",
1945+
);
1946+
yield* Effect.gen(function* () {
1947+
const orchestrator = yield* Orchestrator.OrchestratorV2;
1948+
const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
1949+
const threadId = yield* startFirstTurn;
1950+
const first = started[0]!;
1951+
const turn = (yield* orchestrator.getThreadProjection(threadId)).providerTurns[0]!;
1952+
const turnFailed = yield* orchestrator.streamDomainEvents.pipe(
1953+
Stream.filter(
1954+
(event) =>
1955+
event.type === "provider-turn.updated" &&
1956+
event.payload.id === turn.id &&
1957+
event.payload.status === "failed",
1958+
),
1959+
Stream.take(1),
1960+
Stream.runDrain,
1961+
Effect.forkScoped,
1962+
);
1963+
yield* Queue.offer(events, {
1964+
type: "provider_turn.updated",
1965+
driver,
1966+
providerTurn: { ...turn, status: "failed", completedAt: yield* DateTime.now },
1967+
});
1968+
yield* Fiber.join(turnFailed);
1969+
1970+
yield* orchestrator.dispatch({
1971+
type: "message.dispatch",
1972+
commandId: CommandId.make("restart"),
1973+
threadId,
1974+
messageId: MessageId.make("message:restart"),
1975+
text: "restart",
1976+
attachments: [],
1977+
modelSelection: runSelection,
1978+
dispatchMode: { type: "restart_active", targetRunId: first.runId },
1979+
createdBy: "user",
1980+
creationSource: "web",
1981+
});
1982+
yield* worker.drain();
1983+
1984+
assert.deepEqual(steered, []);
1985+
const projection = yield* orchestrator.getThreadProjection(threadId);
1986+
assert.equal(
1987+
projection.runs.find((run) => run.userMessageId === "message:restart")?.status,
1988+
"queued",
1989+
);
1990+
}).pipe(Effect.provide(layer));
1991+
}),
1992+
),
1993+
);
1994+
17761995
it.effect("queues a plain send behind the stopped queue and releases its hold", () =>
17771996
Effect.scoped(
17781997
Effect.gen(function* () {

0 commit comments

Comments
 (0)