Skip to content

Commit aac2865

Browse files
fix(server): OpenCode 2 turns show provider retries instead of silently running (pingdotgg#15672)
Co-authored-by: Julius Marminge <[email protected]>
1 parent 571d3bc commit aac2865

2 files changed

Lines changed: 369 additions & 2 deletions

File tree

‎apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts‎

Lines changed: 264 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import {
1919
ThreadId,
2020
type ModelSelection,
2121
type OrchestrationV2AppThread,
22+
type OrchestrationV2ProviderFailure,
2223
type OrchestrationV2ProviderThread,
2324
type OrchestrationV2ProviderTurn,
2425
type ProviderReplayEntry,
@@ -2965,7 +2966,11 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => {
29652966
);
29662967
}).pipe(Effect.scoped, Effect.provide(TestClock.layer())),
29672968
);
2968-
const reconnected = (active: Record<string, unknown>, ended?: "succeeded") => [
2969+
const reconnected = (
2970+
active: Record<string, unknown>,
2971+
ended?: "succeeded",
2972+
assistant: Record<string, unknown> = {},
2973+
) => [
29692974
{ type: "runtime_exit", status: "success" } as const,
29702975
out("event.subscribe"),
29712976
out("session.active"),
@@ -3000,6 +3005,7 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => {
30003005
},
30013006
],
30023007
finish: "stop",
3008+
...assistant,
30033009
},
30043010
{
30053011
id: "msg_0eb735d41001NJee1EvVePJAK5",
@@ -3890,6 +3896,263 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => {
38903896
}).pipe(Effect.scoped),
38913897
);
38923898

3899+
const ASSISTANT = "msg_0eb735d5b001oAFVeY5jz3WD4Z";
3900+
const rateLimited = {
3901+
type: "provider.rate-limit",
3902+
message: "Rate limit exceeded: free-models-per-min.",
3903+
status: 429,
3904+
};
3905+
/**
3906+
* OpenCode numbers the request it retries with, so its first retry is
3907+
* attempt 2, and stamps `at` from the clock behind the event's `created` (1 here).
3908+
*/
3909+
const retryScheduled = (attempt: number, delayMs: number) =>
3910+
event("session.retry.scheduled", {
3911+
sessionID: SESSION,
3912+
assistantMessageID: ASSISTANT,
3913+
attempt,
3914+
at: 1 + delayMs,
3915+
error: rateLimited,
3916+
});
3917+
const retrying = {
3918+
class: "provider_error",
3919+
message: rateLimited.message,
3920+
code: rateLimited.type,
3921+
retryable: true,
3922+
} satisfies OrchestrationV2ProviderFailure;
3923+
const firstRetry = { attempt: 1, maxAttempts: null, retryDelayMs: 2_000 };
3924+
const errorRows = (collected: ReadonlyArray<ProviderAdapter.ProviderAdapterV2Event>) =>
3925+
collected.flatMap((event) =>
3926+
event.type === "turn_item.updated" && event.turnItem.type === "error" ? [event.turnItem] : [],
3927+
);
3928+
const runRetryTurn = (entries: ReadonlyArray<ProviderReplayEntry>, gate?: ProviderReplayGate) =>
3929+
Effect.gen(function* () {
3930+
const { runtime, thread } = yield* resumed(
3931+
[
3932+
out("session.prompt", { sessionID: SESSION, text: "<any>" }),
3933+
promptAccepted,
3934+
event("session.execution.started", { sessionID: SESSION }),
3935+
...entries,
3936+
],
3937+
gate === undefined ? undefined : { replayGate: gate },
3938+
);
3939+
const retryShown = yield* Deferred.make<void>();
3940+
const collected = yield* runtime.events.pipe(
3941+
Stream.tap((event) =>
3942+
event.type === "turn_item.updated" && event.turnItem.type === "error"
3943+
? Deferred.succeed(retryShown, undefined)
3944+
: Effect.void,
3945+
),
3946+
Stream.takeUntil((event) => event.type === "turn.terminal"),
3947+
Stream.runCollect,
3948+
Effect.forkScoped,
3949+
);
3950+
yield* runtime.startTurn(turnInput(thread));
3951+
if (gate !== undefined) {
3952+
yield* Deferred.await(retryShown);
3953+
yield* TestClock.adjust("5 seconds");
3954+
gate.releaseAll();
3955+
}
3956+
return yield* Fiber.join(collected);
3957+
});
3958+
3959+
it.effect.each([
3960+
{
3961+
name: "recovers a provider retry at the provider's first output",
3962+
after: [
3963+
event("session.text.started", {
3964+
sessionID: SESSION,
3965+
assistantMessageID: ASSISTANT,
3966+
ordinal: 0,
3967+
}),
3968+
event("session.text.ended", {
3969+
sessionID: SESSION,
3970+
assistantMessageID: ASSISTANT,
3971+
ordinal: 0,
3972+
text: "DONE",
3973+
}),
3974+
event("session.execution.succeeded", { sessionID: SESSION }),
3975+
],
3976+
items: ["error:running", "error:completed", "assistant_message:completed"],
3977+
ended: "completed" as const,
3978+
title: "Provider recovered",
3979+
},
3980+
{
3981+
name: "stops a provider retry when the user stops the turn",
3982+
after: [event("session.execution.interrupted", { sessionID: SESSION, reason: "user" })],
3983+
items: ["error:running", "error:interrupted"],
3984+
ended: "interrupted" as const,
3985+
title: "Provider retry stopped",
3986+
},
3987+
])("$name", ({ after, items, ended, title }) =>
3988+
Effect.gen(function* () {
3989+
const seen = yield* runRetryTurn([retryScheduled(2, 2_000), ...after]);
3990+
assert.deepEqual(turnItems(seen), items);
3991+
const rows = errorRows(seen);
3992+
assert.deepEqual(
3993+
rows.map(({ id, title, failure, retry }) => ({ id, title, failure, retry })),
3994+
[
3995+
{ id: rows[0]!.id, title: "Provider retry", failure: retrying, retry: firstRetry },
3996+
{ id: rows[0]!.id, title, failure: retrying, retry: firstRetry },
3997+
],
3998+
);
3999+
assert.deepInclude(seen.at(-1), { type: "turn.terminal", status: ended, failure: null });
4000+
}).pipe(Effect.scoped),
4001+
);
4002+
4003+
it.effect.each([
4004+
{ name: "restores retry pending at reconnect", pending: true, streamed: false },
4005+
{ name: "settles retry recovered during drop", pending: false, streamed: false },
4006+
{ name: "keeps retry pending when only streamed at reconnect", pending: true, streamed: true },
4007+
])("$name", ({ pending, streamed }) =>
4008+
Effect.gen(function* () {
4009+
const gate = makeProviderReplayGate(["finish"]);
4010+
const { runtime, thread } = yield* resumed(
4011+
[
4012+
out("session.prompt", { sessionID: SESSION, id: PROMPT_ID, text: "<any>" }),
4013+
promptAccepted,
4014+
event("session.execution.started", { sessionID: SESSION }),
4015+
...(pending && !streamed ? [] : [retryScheduled(2, 2_000)]),
4016+
...reconnected({ [SESSION]: { type: "running" } }, undefined, {
4017+
id: ASSISTANT,
4018+
finish: undefined,
4019+
content: pending ? [] : [{ type: "text", text: "Recovered during the drop." }],
4020+
...(pending && !streamed
4021+
? { retry: { attempt: 2, at: 2_001, error: rateLimited } }
4022+
: {}),
4023+
...(streamed ? { time: { created: 2, streamed: 3 } } : {}),
4024+
}),
4025+
...openRequests([shellAsk.data]),
4026+
event("session.execution.succeeded", { sessionID: SESSION }, "finish"),
4027+
],
4028+
{ supervised: true, replayGate: gate },
4029+
);
4030+
const events = yield* runtime.events.pipe(
4031+
Stream.takeUntil((event) => event.type === "turn.terminal"),
4032+
Stream.runCollect,
4033+
Effect.forkScoped,
4034+
);
4035+
yield* runtime.startTurn(turnInput(thread, bigPickle, "approval-required"));
4036+
yield* Effect.promise(() => gate.waitForReached("finish"));
4037+
gate.release("finish");
4038+
const collected = yield* Fiber.join(events);
4039+
// Reconciled requests follow backfill, before the terminal can settle the retry row.
4040+
const reconciled = collected.findIndex((event) => event.type === "runtime_request.updated");
4041+
assert.isAtLeast(reconciled, 0);
4042+
const seen = collected.slice(0, reconciled);
4043+
const rows = errorRows(seen);
4044+
assert.deepEqual(
4045+
rows.map(({ status, title }) => ({ status, title })),
4046+
pending
4047+
? [{ status: "running", title: "Provider retry" }]
4048+
: [
4049+
{ status: "running", title: "Provider retry" },
4050+
{ status: "completed", title: "Provider recovered" },
4051+
],
4052+
);
4053+
for (const row of rows) {
4054+
assert.deepEqual(row.failure, retrying);
4055+
assert.deepInclude(row.retry, { attempt: 1, maxAttempts: null });
4056+
assert.equal(row.id, rows[0]!.id);
4057+
assert.equal(row.ordinal, rows[0]!.ordinal);
4058+
assert.deepEqual(row.startedAt, rows[0]!.startedAt);
4059+
}
4060+
if (!pending) {
4061+
assert.isTrue(
4062+
seen.some(
4063+
(event) =>
4064+
event.type === "turn_item.updated" &&
4065+
event.turnItem.type === "assistant_message" &&
4066+
event.turnItem.text === "Recovered during the drop.",
4067+
),
4068+
);
4069+
}
4070+
}).pipe(Effect.scoped),
4071+
);
4072+
4073+
it.effect("replaces a provider retry with the error that failed the turn", () =>
4074+
Effect.gen(function* () {
4075+
const step = {
4076+
sessionID: SESSION,
4077+
assistantMessageID: ASSISTANT,
4078+
agent: "build",
4079+
model: { providerID: "opencode", id: "big-pickle", variant: "default" },
4080+
started: 1,
4081+
};
4082+
const gate = makeProviderReplayGate(["second retry"]);
4083+
// OpenCode starts the step of each attempt that fails, so a step start is no recovery.
4084+
const seen = yield* runRetryTurn(
4085+
[
4086+
event("session.step.started", step),
4087+
retryScheduled(2, 2_000),
4088+
event("session.step.started", step, "second retry"),
4089+
retryScheduled(3, 4_000),
4090+
event("session.step.started", step),
4091+
event("session.step.failed", {
4092+
sessionID: SESSION,
4093+
assistantMessageID: ASSISTANT,
4094+
error: rateLimited,
4095+
}),
4096+
event("session.execution.failed", { sessionID: SESSION, error: rateLimited }),
4097+
],
4098+
gate,
4099+
);
4100+
const failed = { ...retrying, class: "usage_limit" as const, retryable: null };
4101+
const lastRetry = { attempt: 2, maxAttempts: null, retryDelayMs: 4_000 };
4102+
const rows = errorRows(seen);
4103+
assert.deepEqual(
4104+
rows.map(({ id, status, title, failure, retry }) => ({
4105+
id,
4106+
status,
4107+
title,
4108+
failure,
4109+
retry,
4110+
})),
4111+
[
4112+
{
4113+
id: rows[0]!.id,
4114+
status: "running",
4115+
title: "Provider retry",
4116+
failure: retrying,
4117+
retry: firstRetry,
4118+
},
4119+
{
4120+
id: rows[0]!.id,
4121+
status: "running",
4122+
title: "Provider retry",
4123+
failure: retrying,
4124+
retry: lastRetry,
4125+
},
4126+
{
4127+
id: rows[0]!.id,
4128+
status: "failed",
4129+
title: "Usage limit reached",
4130+
failure: failed,
4131+
retry: lastRetry,
4132+
},
4133+
],
4134+
);
4135+
const { startedAt, ordinal } = rows[0]!;
4136+
assert.ok(startedAt);
4137+
assert.deepEqual(
4138+
rows.map((row) => ({ startedAt: row.startedAt, ordinal: row.ordinal })),
4139+
rows.map(() => ({ startedAt, ordinal })),
4140+
);
4141+
assert.isAtLeast(
4142+
DateTime.toEpochMillis(rows[2]!.updatedAt) - DateTime.toEpochMillis(startedAt),
4143+
5_000,
4144+
);
4145+
assert.deepInclude(seen.at(-1), {
4146+
type: "turn.terminal",
4147+
status: "failed",
4148+
failure: failed,
4149+
failureItemOrdinal: ordinal,
4150+
retry: lastRetry,
4151+
retryStartedAt: startedAt,
4152+
});
4153+
}).pipe(Effect.scoped),
4154+
);
4155+
38934156
it.effect("reports a model's input limit as its context window", () =>
38944157
Effect.gen(function* () {
38954158
const runtime = yield* openCode2ReplayRuntime([...opening]);

0 commit comments

Comments
 (0)