Skip to content

Commit 945c5ef

Browse files
committed
fix(server): settle force-stopped background work during checkpointing
Use the existing active-run predicate and background settlement for a waiting run, retaining the exact-attempt guard and run-ordinal bounds. Leave checkpoint capture, root completion, and run finalization untouched. Add a regression that fails on the PR base, checks stale-attempt isolation, and preserves the pending checkpoint effect. Verified check:changed, lifecycle/finalization suites, and fresh independent review.
1 parent 062b32f commit 945c5ef

2 files changed

Lines changed: 140 additions & 2 deletions

File tree

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11871,8 +11871,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1187111871
}
1187211872
if (
1187311873
incoming.type === "run.force-stop.settle" &&
11874-
(run.activeAttemptId !== incoming.attemptId ||
11875-
!["preparing", "starting", "running"].includes(run.status))
11874+
(run.activeAttemptId !== incoming.attemptId || !isActiveRun(run))
1187611875
)
1187711876
return undefined;
1187811877
const ownership =
@@ -11888,6 +11887,17 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1188811887
Option.isSome(ownership) &&
1188911888
(providerTurn === undefined || ownership.value.startPending);
1189011889
const now = yield* DateTime.now;
11890+
if (incoming.type === "run.force-stop.settle" && run.status === "waiting") {
11891+
yield* settleBackgroundWork({
11892+
command,
11893+
events,
11894+
projection,
11895+
stoppedProviderThreadId: providerThread.id,
11896+
throughRunOrdinal: run.ordinal,
11897+
now,
11898+
});
11899+
return undefined;
11900+
}
1189111901
const completionMessage = projection.messages.find(
1189211902
(candidate) => candidate.id === run.userMessageId,
1189311903
);

‎apps/server/src/orchestration-v2/ThreadStop.test.ts‎

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { assert, it } from "@effect/vitest";
22
import {
3+
CheckpointScopeId,
34
CommandId,
45
ContextTransferId,
56
EventId,
@@ -13,6 +14,7 @@ import {
1314
ProviderThreadId,
1415
ProviderTurnId,
1516
RuntimeRequestId,
17+
RunAttemptId,
1618
ScheduledTaskId,
1719
ThreadId,
1820
TurnItemId,
@@ -21,6 +23,7 @@ import * as DateTime from "effect/DateTime";
2123
import * as Effect from "effect/Effect";
2224
import * as Exit from "effect/Exit";
2325
import * as Layer from "effect/Layer";
26+
import * as Option from "effect/Option";
2427
import * as Schema from "effect/Schema";
2528
import * as SqlClient from "effect/sql/SqlClient";
2629
import * as Stream from "effect/Stream";
@@ -1553,6 +1556,131 @@ it.effect.each(["missing session", "completed", "failed", "cancelled"] as const)
15531556
}).pipe(Effect.provide(layerTest)),
15541557
);
15551558

1559+
it.effect("Force stop settles background work without interrupting checkpoint capture", () =>
1560+
Effect.gen(function* () {
1561+
const orchestrator = yield* Orchestrator.OrchestratorV2;
1562+
const projections = yield* ProjectionStore.ProjectionStoreV2;
1563+
const outbox = yield* EffectOutbox.EffectOutboxV2;
1564+
const threadId = ThreadId.make("thread:force-stop-checkpoint");
1565+
yield* createWatchingThread(threadId, 21);
1566+
yield* send(threadId, "work", "start_immediately");
1567+
yield* delegate(threadId, "native-background");
1568+
const before = yield* orchestrator.getThreadProjection(threadId);
1569+
const run = before.runs[0]!;
1570+
const attempt = before.attempts[0]!;
1571+
const root = before.nodes.find((node) => node.id === run.rootNodeId)!;
1572+
const subagent = before.subagents[0]!;
1573+
const item = before.turnItems.find((item) => item.type === "subagent")!;
1574+
const now = yield* DateTime.now;
1575+
const scopeId = CheckpointScopeId.make("scope:force-stop-checkpoint");
1576+
const providerTurnId = ProviderTurnId.make("turn:force-stop-checkpoint");
1577+
const apply = (event: Omit<OrchestrationV2DomainEvent, "id" | "threadId">) =>
1578+
projections.apply({
1579+
...event,
1580+
id: EventId.make(`event:force-stop-checkpoint:${event.type}`),
1581+
threadId,
1582+
} as OrchestrationV2DomainEvent);
1583+
yield* apply({
1584+
type: "run.updated",
1585+
runId: run.id,
1586+
occurredAt: now,
1587+
payload: { ...run, status: "waiting", startedAt: now },
1588+
});
1589+
yield* apply({
1590+
type: "run-attempt.updated",
1591+
occurredAt: now,
1592+
payload: { ...attempt, status: "completed", startedAt: now, completedAt: now },
1593+
});
1594+
yield* apply({
1595+
type: "node.updated",
1596+
nodeId: root.id,
1597+
occurredAt: now,
1598+
payload: { ...root, status: "completed", checkpointScopeId: scopeId, completedAt: now },
1599+
});
1600+
yield* apply({
1601+
type: "provider-turn.updated",
1602+
occurredAt: now,
1603+
payload: {
1604+
id: providerTurnId,
1605+
providerThreadId: run.providerThreadId!,
1606+
nodeId: root.id,
1607+
runAttemptId: attempt.id,
1608+
nativeTurnRef: null,
1609+
ordinal: 1,
1610+
status: "completed",
1611+
startedAt: now,
1612+
completedAt: now,
1613+
},
1614+
});
1615+
yield* apply({
1616+
type: "subagent.updated",
1617+
nodeId: subagent.id,
1618+
occurredAt: now,
1619+
payload: { ...subagent, origin: "provider_native", childThreadId: null, status: "running" },
1620+
});
1621+
yield* apply({
1622+
type: "turn-item.updated",
1623+
runId: run.id,
1624+
occurredAt: now,
1625+
payload: {
1626+
...item,
1627+
origin: "provider_native",
1628+
childThreadId: null,
1629+
providerThreadId: run.providerThreadId,
1630+
providerTurnId,
1631+
status: "running",
1632+
},
1633+
});
1634+
const captureId = `effect:checkpoint.capture:${run.id}`;
1635+
yield* outbox.enqueue([
1636+
{
1637+
id: captureId,
1638+
commandId: CommandId.make("checkpoint:force-stop-checkpoint"),
1639+
threadId,
1640+
request: { type: "checkpoint.capture", runId: run.id, scopeId },
1641+
},
1642+
]);
1643+
const captureBefore = Option.getOrThrow(yield* outbox.get(captureId));
1644+
assert.equal(captureBefore.status, "pending");
1645+
const waiting = yield* orchestrator.getThreadProjection(threadId);
1646+
const settle = {
1647+
type: "run.force-stop.settle" as const,
1648+
threadId,
1649+
runId: run.id,
1650+
attemptId: attempt.id,
1651+
reason: "Force stopped by user",
1652+
};
1653+
const stale = yield* orchestrator.dispatch({
1654+
...settle,
1655+
commandId: CommandId.make("force-stop-checkpoint:stale-attempt"),
1656+
attemptId: RunAttemptId.make("attempt:stale"),
1657+
});
1658+
assert.lengthOf(stale.storedEvents, 0);
1659+
assert.deepEqual(yield* orchestrator.getThreadProjection(threadId), waiting);
1660+
1661+
yield* orchestrator.dispatch({
1662+
...settle,
1663+
commandId: CommandId.make("force-stop-checkpoint:settle"),
1664+
});
1665+
const settled = yield* orchestrator.getThreadProjection(threadId);
1666+
assert.equal(
1667+
settled.turnItems.find((candidate) => candidate.id === item.id)?.status,
1668+
"interrupted",
1669+
);
1670+
assert.equal(
1671+
settled.subagents.find((candidate) => candidate.id === subagent.id)?.status,
1672+
"interrupted",
1673+
);
1674+
assert.deepEqual(settled.runs, waiting.runs);
1675+
assert.deepEqual(settled.attempts, waiting.attempts);
1676+
assert.deepEqual(
1677+
settled.nodes.find((node) => node.id === root.id),
1678+
waiting.nodes.find((node) => node.id === root.id),
1679+
);
1680+
assert.deepEqual(Option.getOrThrow(yield* outbox.get(captureId)), captureBefore);
1681+
}).pipe(Effect.provide(layerTest)),
1682+
);
1683+
15561684
it.effect("an unanswered Stop is flagged until Force stop ends the run with its own reason", () =>
15571685
Effect.gen(function* () {
15581686
const orchestrator = yield* Orchestrator.OrchestratorV2;

0 commit comments

Comments
 (0)