Skip to content

Commit b368b81

Browse files
committed
fix(opencode): preserve shell projections after recovery timeouts
1 parent dfc152b commit b368b81

2 files changed

Lines changed: 197 additions & 40 deletions

File tree

‎apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts‎

Lines changed: 178 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -1214,7 +1214,7 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => {
12141214
}).pipe(Effect.scoped),
12151215
);
12161216

1217-
it.effect.each(["remove", "lookup", "output"] as const)(
1217+
it.effect.each(["remove", "lookup", "output", "timeout"] as const)(
12181218
"continues recovering and stopping native work when shell %s fails",
12191219
(failure) =>
12201220
Effect.gen(function* () {
@@ -1234,7 +1234,8 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => {
12341234
out("shell.output", { id }),
12351235
reply("shell.output", response({ output: "", cursor: 0, size: 0, truncated: false })),
12361236
];
1237-
const { runtime, thread } = yield* resumed(
1237+
const gate = makeProviderReplayGate(["recovery-pending", "polled"]);
1238+
const recovery = yield* resumed(
12381239
[
12391240
...backgroundLaunch(CHILD),
12401241
...output(ids[0]!),
@@ -1258,29 +1259,74 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => {
12581259
],
12591260
{
12601261
version: "2.2.1",
1262+
replayGate: gate,
12611263
shellRecovery: [
12621264
out("shell.list", "<any>"),
12631265
reply("shell.list", response(ids.map(info))),
1264-
...ids.flatMap((id) => [
1265-
out("shell.get", { id }),
1266-
reply(
1267-
"shell.get",
1268-
id === ids[0] && failure === "lookup"
1269-
? { status: 500, body: {} }
1270-
: response(info(id)),
1271-
),
1272-
...(id === ids[0] && failure === "lookup"
1273-
? []
1274-
: id === ids[0] && failure === "output"
1275-
? [
1276-
out("shell.output", { id }),
1277-
reply("shell.output", { status: 500, body: {} }),
1278-
]
1279-
: output(id)),
1280-
]),
1266+
...(failure === "timeout"
1267+
? [
1268+
out("shell.get", { id: ids[0] }),
1269+
reply("shell.get", "<hang>"),
1270+
event(
1271+
"session.execution.succeeded",
1272+
{ sessionID: "ses_unrelated" },
1273+
"recovery-pending",
1274+
),
1275+
...ids.map((id) => out("shell.output", { id })),
1276+
...ids.map(() =>
1277+
reply(
1278+
"shell.output",
1279+
response({ output: "", cursor: 0, size: 0, truncated: false }),
1280+
),
1281+
),
1282+
event("session.execution.succeeded", { sessionID: "ses_unrelated" }, "polled"),
1283+
]
1284+
: ids.flatMap((id) => [
1285+
out("shell.get", { id }),
1286+
reply(
1287+
"shell.get",
1288+
id === ids[0] && failure === "lookup"
1289+
? { status: 500, body: {} }
1290+
: response(info(id)),
1291+
),
1292+
...(id === ids[0] && failure === "lookup"
1293+
? []
1294+
: id === ids[0] && failure === "output"
1295+
? [
1296+
out("shell.output", { id }),
1297+
reply("shell.output", { status: 500, body: {} }),
1298+
]
1299+
: output(id)),
1300+
])),
12811301
],
12821302
},
1303+
).pipe(Effect.forkScoped);
1304+
if (failure === "timeout") {
1305+
yield* Effect.promise(() => gate.waitForReached("recovery-pending"));
1306+
yield* TestClock.adjust("5 seconds");
1307+
}
1308+
const { runtime, thread } = yield* Fiber.join(recovery);
1309+
const recovered = yield* runtime.events.pipe(
1310+
Stream.takeUntil(
1311+
(event) =>
1312+
event.type === "provider_thread.updated" &&
1313+
event.providerThread.pendingBackgroundTasks?.length === 2,
1314+
),
1315+
Stream.runCollect,
12831316
);
1317+
assert.sameMembers(
1318+
recovered.flatMap((event) =>
1319+
event.type === "turn_item.updated" && event.turnItem.type === "command_execution"
1320+
? [event.turnItem.nativeItemRef?.nativeId]
1321+
: [],
1322+
),
1323+
ids,
1324+
);
1325+
if (failure === "timeout") {
1326+
gate.release("recovery-pending");
1327+
yield* Effect.promise(() => gate.waitForReached("polled"));
1328+
gate.release("polled");
1329+
}
12841330
const watch = yield* watchBackgroundTurn(runtime);
12851331
yield* runtime.startTurn(withLineage(thread));
12861332
yield* watch.attached;
@@ -1587,26 +1633,119 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => {
15871633
event("session.execution.started", { sessionID: CHILD }),
15881634
];
15891635

1590-
it.effect("settles a background subagent that fails, so no work is left pending", () =>
1591-
Effect.gen(function* () {
1592-
const { runtime, thread } = yield* resumed([
1593-
...launchedAndRunning,
1594-
// The parent turn ends, and the background child fails: it reports
1595-
// nothing back, so its call has to end with the failure.
1596-
event("session.execution.succeeded", { sessionID: SESSION }),
1597-
event("session.execution.failed", {
1598-
sessionID: CHILD,
1599-
error: { type: "provider.quota", message: "Rate limit exceeded." },
1600-
}),
1601-
]);
1602-
const watch = yield* watchCalls(runtime, thread);
1603-
yield* runtime.startTurn(withLineage(thread));
1604-
yield* watch.until(
1605-
() => watch.calls.get("Sleep")?.status === "failed" && watch.state.roster?.length === 0,
1606-
);
1607-
assert.deepEqual(watch.state.roster, []);
1608-
assert.isFalse(yield* runtime.hasPendingBackgroundWorkForThread!(thread));
1609-
}).pipe(Effect.scoped),
1636+
it.effect.each([false, true])(
1637+
"settles a failed background subagent with recovered child=%s",
1638+
(recoveredChild) =>
1639+
Effect.gen(function* () {
1640+
const ids = yield* IdAllocator.IdAllocatorV2.pipe(Effect.provide(IdAllocator.layer));
1641+
const childThreadId = ids.derive.threadFromProviderThread({
1642+
driver: OPENCODE_PROVIDER,
1643+
nativeThreadId: CHILD,
1644+
providerInstanceId: instanceId,
1645+
});
1646+
const childProviderThread = {
1647+
...providerThread(yield* DateTime.now),
1648+
id: ids.derive.providerThread({
1649+
driver: OPENCODE_PROVIDER,
1650+
nativeThreadId: CHILD,
1651+
providerInstanceId: instanceId,
1652+
}),
1653+
appThreadId: childThreadId,
1654+
nativeThreadRef: {
1655+
driver: OPENCODE_PROVIDER,
1656+
nativeId: CHILD,
1657+
strength: "strong" as const,
1658+
},
1659+
};
1660+
const shell = {
1661+
id: "sh_old_child",
1662+
command: "echo done",
1663+
status: "exited",
1664+
exit: 0,
1665+
cwd: WORK,
1666+
shell: "/bin/sh",
1667+
file: "/remote/output",
1668+
metadata: { sessionID: CHILD },
1669+
time: { started: 1, completed: 2 },
1670+
};
1671+
const response = (data: unknown) => ({ location: { directory: WORK }, data });
1672+
const { runtime, thread } = yield* resumed(
1673+
[
1674+
...(recoveredChild
1675+
? [
1676+
out("session.get", { sessionID: CHILD }),
1677+
replyData("session.get", sessionInfo({ id: CHILD })),
1678+
out("permission.list", { sessionID: CHILD }),
1679+
replyData("permission.list", []),
1680+
out("session.form.list", { sessionID: CHILD }),
1681+
replyData("session.form.list", []),
1682+
out("shell.list", "<any>"),
1683+
reply("shell.list", response([shell])),
1684+
out("shell.get", { id: shell.id }),
1685+
reply("shell.get", response(shell)),
1686+
out("shell.output", { id: shell.id }),
1687+
reply(
1688+
"shell.output",
1689+
response({ output: "done", cursor: 4, size: 4, truncated: false }),
1690+
),
1691+
out("session.update", { sessionID: CHILD, permissions: "<any>" }),
1692+
reply("session.update", null),
1693+
]
1694+
: []),
1695+
...launchedAndRunning,
1696+
// The parent turn ends, and the background child fails: it reports
1697+
// nothing back, so its call has to end with the failure.
1698+
event("session.execution.succeeded", { sessionID: SESSION }),
1699+
event("session.execution.failed", {
1700+
sessionID: CHILD,
1701+
error: { type: "provider.quota", message: "Rate limit exceeded." },
1702+
}),
1703+
],
1704+
recoveredChild ? { version: "2.2.1" } : undefined,
1705+
);
1706+
if (recoveredChild)
1707+
yield* runtime.resumeThread({
1708+
providerThread: childProviderThread,
1709+
threadId: childThreadId,
1710+
modelSelection: bigPickle,
1711+
runtimePolicy: policy(),
1712+
});
1713+
const childUpdates: Array<
1714+
Extract<ProviderAdapter.ProviderAdapterV2Event, { type: "provider_thread.updated" }>
1715+
> = [];
1716+
const watch = yield* watchCalls(
1717+
{
1718+
...runtime,
1719+
events: runtime.events.pipe(
1720+
Stream.tap((event) =>
1721+
Effect.sync(() => {
1722+
if (
1723+
event.type === "provider_thread.updated" &&
1724+
event.providerThread.id === childProviderThread.id &&
1725+
event.providerThread.ownerNodeId !== null
1726+
)
1727+
childUpdates.push(event);
1728+
}),
1729+
),
1730+
),
1731+
},
1732+
thread,
1733+
);
1734+
yield* runtime.startTurn(withLineage(thread));
1735+
yield* watch.until(
1736+
() => watch.calls.get("Sleep")?.status === "failed" && watch.state.roster?.length === 0,
1737+
);
1738+
assert.includeMembers(
1739+
childUpdates.map((event) => event.providerThread.status),
1740+
["active", "idle"],
1741+
);
1742+
assert.isFalse(
1743+
childUpdates.some((event) => event.sessionScoped === true),
1744+
"reattached child status must route through the parent run",
1745+
);
1746+
assert.deepEqual(watch.state.roster, []);
1747+
assert.isFalse(yield* runtime.hasPendingBackgroundWorkForThread!(thread));
1748+
}).pipe(Effect.scoped),
16101749
);
16111750

16121751
// regression: #21979 a failed background subagent pinned its thread as busy

‎packages/provider-opencode/src/server/v2/adapter.ts‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -906,6 +906,7 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr
906906
cursor: number;
907907
startedAt?: DateTime.Utc;
908908
recoveredRunId?: RunId | null;
909+
published?: boolean;
909910
}
910911
>();
911912

@@ -1637,6 +1638,7 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr
16371638
...(exit === undefined ? {} : { exitCode: exit }),
16381639
},
16391640
});
1641+
shell.published = true;
16401642
return;
16411643
}
16421644
shell.turn.tools.set(shell.toolId, shell.tool);
@@ -1646,6 +1648,7 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr
16461648
metadata: { shellID: id, exit },
16471649
});
16481650
shell.turn.tools.delete(shell.toolId);
1651+
shell.published = true;
16491652
});
16501653

16511654
const stopShell = Effect.fn("OpenCode2Adapter.stopShell")(function* (id: string) {
@@ -1776,7 +1779,19 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr
17761779
lock.withPermit(
17771780
effect.pipe(
17781781
Effect.timeout(INVENTORY_TIMEOUT),
1779-
Effect.ensuring(syncRosterAncestors(state)),
1782+
Effect.ensuring(
1783+
Effect.gen(function* () {
1784+
for (const [id, shell] of shells) {
1785+
if (
1786+
shell.running &&
1787+
shell.published !== true &&
1788+
sessionsOf(state).includes(shell.state)
1789+
)
1790+
yield* emitShell(id, "running");
1791+
}
1792+
yield* syncRosterAncestors(state);
1793+
}),
1794+
),
17801795
),
17811796
),
17821797
);
@@ -1879,6 +1894,7 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr
18791894
const child =
18801895
previous ?? newThreadState(childId, providerThread, call.state.directory, subagent);
18811896
child.subagent = subagent;
1897+
child.sessionScopedBackground = false;
18821898
child.model = info?.model ?? child.model;
18831899
child.providerThread = withReportedModel(providerThread, child.model);
18841900
child.directory = call.state.directory;
@@ -1918,6 +1934,7 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr
19181934
if (child.active !== undefined && status !== "completed") {
19191935
yield* finishTurn(child, { status: "interrupted" });
19201936
}
1937+
child.sessionScopedBackground = false;
19211938
child.providerThread = {
19221939
...child.providerThread,
19231940
status: "idle",
@@ -1965,6 +1982,7 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr
19651982
turn.before = subagent.queued;
19661983
subagent.queued = undefined;
19671984
child.active = turn;
1985+
child.sessionScopedBackground = false;
19681986
yield* emit({
19691987
type: "node.updated",
19701988
driver,

0 commit comments

Comments
 (0)