Skip to content

Commit 807319c

Browse files
authored
fix(opencode): retry failed model catalogs until they recover (#23939)
1 parent 49597c7 commit 807319c

7 files changed

Lines changed: 491 additions & 24 deletions

File tree

‎packages/provider-core/src/server/managedProvider.test.ts‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import * as Layer from "effect/Layer";
1313
import * as PubSub from "effect/PubSub";
1414
import * as Ref from "effect/Ref";
1515
import * as Stream from "effect/Stream";
16+
import * as TestClock from "effect/testing/TestClock";
1617

1718
import * as ProviderHost from "./ProviderHost.ts";
1819
import { makeManagedServerProvider } from "./managedProvider.ts";
@@ -179,6 +180,80 @@ describe("makeManagedServerProvider", () => {
179180
).pipe(Effect.provide(layerAlwaysRunTest)),
180181
);
181182

183+
it.effect("skips periodic provider refreshes without foreground provider-status demand", () =>
184+
Effect.scoped(
185+
Effect.gen(function* () {
186+
const checkCalls = yield* Ref.make(0);
187+
const initialCheckDone = yield* Deferred.make<void>();
188+
yield* makeManagedServerProvider<TestSettings>({
189+
resolveMaintenance: () => Effect.succeed(maintenanceCapabilities),
190+
getSettings: Effect.succeed({ enabled: true }),
191+
streamSettings: Stream.empty,
192+
haveSettingsChanged: (previous, next) => previous.enabled !== next.enabled,
193+
refreshOnInterval: () => true,
194+
initialSnapshot: () => Effect.succeed(initialSnapshot),
195+
checkProvider: Ref.updateAndGet(checkCalls, (count) => count + 1).pipe(
196+
Effect.tap((count) =>
197+
count === 1
198+
? Deferred.succeed(initialCheckDone, undefined).pipe(Effect.ignore)
199+
: Effect.void,
200+
),
201+
Effect.as(refreshedSnapshot),
202+
),
203+
refreshInterval: "1 second",
204+
});
205+
206+
yield* Deferred.await(initialCheckDone);
207+
yield* TestClock.adjust("1 second");
208+
yield* Effect.yieldNow;
209+
210+
assert.strictEqual(yield* Ref.get(checkCalls), 1);
211+
}),
212+
).pipe(
213+
Effect.provide(
214+
Layer.mergeAll(layerProviderHost({ runBackgroundWork: false }), TestClock.layer()),
215+
),
216+
),
217+
);
218+
219+
it.effect("stops periodic recovery checks once the snapshot is ready", () =>
220+
Effect.scoped(
221+
Effect.gen(function* () {
222+
const checkCalls = yield* Ref.make(0);
223+
const releaseInitialCheck = yield* Deferred.make<void>();
224+
const failedSnapshot = { ...refreshedSnapshot, status: "error" as const };
225+
const provider = yield* makeManagedServerProvider<TestSettings>({
226+
resolveMaintenance: () => Effect.succeed(maintenanceCapabilities),
227+
getSettings: Effect.succeed({ enabled: true }),
228+
streamSettings: Stream.empty,
229+
haveSettingsChanged: (previous, next) => previous.enabled !== next.enabled,
230+
refreshOnInterval: (snapshot) => snapshot.status === "error",
231+
initialSnapshot: () => Effect.succeed(initialSnapshot),
232+
checkProvider: Deferred.await(releaseInitialCheck).pipe(
233+
Effect.andThen(Ref.updateAndGet(checkCalls, (count) => count + 1)),
234+
Effect.map((count) => (count === 1 ? failedSnapshot : refreshedSnapshot)),
235+
),
236+
refreshInterval: "1 second",
237+
});
238+
const updates = yield* provider.streamChanges.pipe(
239+
Stream.take(2),
240+
Stream.runCollect,
241+
Effect.forkChild,
242+
);
243+
yield* Effect.yieldNow;
244+
yield* Deferred.succeed(releaseInitialCheck, undefined);
245+
yield* TestClock.adjust("1 second");
246+
247+
assert.deepStrictEqual(Array.from(yield* Fiber.join(updates)), [
248+
failedSnapshot,
249+
refreshedSnapshot,
250+
]);
251+
yield* TestClock.adjust("3 seconds");
252+
assert.strictEqual(yield* Ref.get(checkCalls), 2);
253+
}),
254+
).pipe(Effect.provide(Layer.mergeAll(layerAlwaysRunTest, TestClock.layer()))),
255+
);
256+
182257
it.effect("reruns the provider check when streamed settings change", () =>
183258
Effect.scoped(
184259
Effect.gen(function* () {

‎packages/provider-core/src/server/managedProvider.ts‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ export const makeManagedServerProvider = Effect.fn("makeManagedServerProvider")(
5151
readonly publishSnapshot: (snapshot: ServerProvider) => Effect.Effect<void>;
5252
}) => Effect.Effect<void>;
5353
readonly refreshInterval?: Duration.Input;
54-
readonly refreshOnInterval?: boolean;
54+
readonly refreshOnInterval?: boolean | ((snapshot: ServerProvider) => boolean);
5555
readonly checkProviderOnSettingsChange?: (previous: Settings, next: Settings) => boolean;
5656
}): Effect.fn.Return<
5757
ManagedServerProvider,
@@ -218,11 +218,15 @@ export const makeManagedServerProvider = Effect.fn("makeManagedServerProvider")(
218218
return yield* applySnapshot(nextSettings, { forceRefresh: true });
219219
});
220220

221-
const hasProviderStatusDemand = Effect.gen(function* () {
221+
const shouldRefreshOnInterval = Effect.gen(function* () {
222222
const state = yield* Ref.get(snapshotStateRef);
223223
// A disabled instance has nothing to probe; a periodic check would only
224224
// republish a snapshot that differs in checkedAt.
225-
if (!state.snapshot.enabled) {
225+
if (
226+
!state.snapshot.enabled ||
227+
input.refreshOnInterval === false ||
228+
(typeof input.refreshOnInterval === "function" && !input.refreshOnInterval(state.snapshot))
229+
) {
226230
return false;
227231
}
228232
const instanceId = state.snapshot.instanceId;
@@ -276,10 +280,8 @@ export const makeManagedServerProvider = Effect.fn("makeManagedServerProvider")(
276280
Queue.take(refreshIntervalChanges).pipe(Effect.as(false)),
277281
).pipe(
278282
Effect.flatMap((intervalElapsed) =>
279-
input.refreshOnInterval !== false &&
280-
intervalElapsed &&
281-
Duration.toMillis(Duration.fromInputUnsafe(refreshInterval)) > 0
282-
? hasProviderStatusDemand.pipe(
283+
intervalElapsed && Duration.toMillis(Duration.fromInputUnsafe(refreshInterval)) > 0
284+
? shouldRefreshOnInterval.pipe(
283285
Effect.flatMap((shouldRefresh) =>
284286
shouldRefresh ? refreshSnapshot().pipe(Effect.asVoid) : Effect.void,
285287
),

‎packages/provider-opencode/src/server/driver.test.ts‎

Lines changed: 170 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,11 +8,16 @@ import { assert, it } from "@effect/vitest";
88
import { ProviderInstanceId } from "@t3tools/contracts";
99
import type { OpenCodeSettings } from "../settings.ts";
1010
import * as HostProcess from "@t3tools/shared/HostProcess";
11+
import * as Deferred from "effect/Deferred";
1112
import * as Effect from "effect/Effect";
1213
import * as Fiber from "effect/Fiber";
1314
import * as Layer from "effect/Layer";
15+
import * as Logger from "effect/Logger";
16+
import * as References from "effect/References";
17+
import * as Stream from "effect/Stream";
1418
import * as TestClock from "effect/testing/TestClock";
1519
import * as HttpClient from "effect/http/HttpClient";
20+
import * as HttpClientResponse from "effect/http/HttpClientResponse";
1621
import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner";
1722

1823
import * as ProviderLatestVersions from "@t3tools/provider-core/server/ProviderLatestVersions";
@@ -21,6 +26,7 @@ import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderS
2126
import * as ProviderEventLoggers from "@t3tools/provider-core/server/ProviderEventLoggers";
2227
import * as ProviderMaintenance from "@t3tools/provider-core/server/maintenanceResolver";
2328
import * as OpenCodeRuntime from "./OpenCodeRuntime.ts";
29+
import { OPENCODE_2_RESPONSES, replayOpenCodeServer } from "./probeResponses.fixture.ts";
2430
import { OpenCodeDriver, openCodeUpdateFor } from "./driver.ts";
2531
import * as TestProviderHost from "@t3tools/provider-testing/TestProviderHost";
2632

@@ -94,6 +100,170 @@ it.layer(layer)("OpenCodeDriver runtime selection", (it) => {
94100
yield* Fiber.join(capabilities);
95101
}).pipe(Effect.scoped, Effect.provide(TestClock.layer())),
96102
);
103+
104+
it.effect.each(["probe", "error", "empty"] as const)(
105+
"recovers discovery without a manual refresh after %s",
106+
(initialCatalog) =>
107+
Effect.gen(function* () {
108+
const releaseCatalog = yield* Deferred.make<void>();
109+
const discoveryLogs: Array<Record<string, unknown>> = [];
110+
const logger = Logger.make(({ message, fiber }) => {
111+
if (Array.isArray(message) && message.includes("OpenCode discovery failed")) {
112+
discoveryLogs.push({ ...fiber.getRef(References.CurrentLogAnnotations) });
113+
}
114+
});
115+
let catalogAvailable = false;
116+
let modelReads = 0;
117+
const replies = {
118+
...OPENCODE_2_RESPONSES,
119+
"/api/provider": {
120+
status: 200,
121+
contentType: "application/json",
122+
body: '{"location":{"directory":"/work"},"data":[]}',
123+
},
124+
"/api/model": {
125+
status: 200,
126+
contentType: "application/json",
127+
body: JSON.stringify({
128+
location: { directory: "/work" },
129+
data: [
130+
{
131+
id: "big-pickle",
132+
modelID: "big-pickle",
133+
providerID: "opencode",
134+
name: "Big Pickle",
135+
package: "@opencode/ai/providers/openai-compatible",
136+
settings: {},
137+
compatibility: {},
138+
capabilities: { tools: true, input: ["text"], output: ["text"] },
139+
variants: [],
140+
time: { released: 1789516800000 },
141+
cost: [{ input: 0, output: 0, cache: { read: 0, write: 0 } }],
142+
status: "active",
143+
enabled: true,
144+
limit: { context: 1000000, output: 65536 },
145+
},
146+
],
147+
}),
148+
},
149+
};
150+
const replay = replayOpenCodeServer(replies, "secret");
151+
const server = HttpClient.make((request) =>
152+
Effect.gen(function* () {
153+
if (initialCatalog === "probe" && !catalogAvailable) {
154+
yield* Deferred.await(releaseCatalog);
155+
return HttpClientResponse.fromWeb(
156+
request,
157+
new Response('{"_tag":"InternalServerError","message":"server starting"}', {
158+
status: 503,
159+
headers: { "content-type": "application/json" },
160+
}),
161+
);
162+
}
163+
if (new URL(request.url).pathname === "/api/model") {
164+
modelReads += 1;
165+
yield* Deferred.await(releaseCatalog);
166+
if (!catalogAvailable) {
167+
return HttpClientResponse.fromWeb(
168+
request,
169+
new Response(
170+
initialCatalog === "error"
171+
? '{"_tag":"InternalServerError","message":"catalog unavailable"}'
172+
: '{"location":{"directory":"/work"},"data":[]}',
173+
{
174+
status: initialCatalog === "error" ? 500 : 200,
175+
headers: { "content-type": "application/json" },
176+
},
177+
),
178+
);
179+
}
180+
}
181+
return yield* replay.execute(request);
182+
}),
183+
);
184+
const instance = yield* create(
185+
{ serverUrl: "http://127.0.0.1:4096", serverPassword: "secret" },
186+
server,
187+
).pipe(Effect.provide(Logger.layer([logger], { mergeWithExisting: false })));
188+
const initial = yield* instance.snapshot.streamChanges.pipe(
189+
Stream.filter(
190+
(snapshot) =>
191+
snapshot.installed &&
192+
snapshot.status === (initialCatalog === "empty" ? "warning" : "error"),
193+
),
194+
Stream.take(1),
195+
Stream.runCollect,
196+
Effect.forkChild,
197+
);
198+
yield* Effect.yieldNow;
199+
yield* Deferred.succeed(releaseCatalog, undefined);
200+
yield* TestClock.adjust("5 seconds");
201+
const [failed] = Array.from(yield* Fiber.join(initial));
202+
assert.strictEqual(failed?.auth.status, "unknown");
203+
assert.deepStrictEqual(failed?.models, []);
204+
if (initialCatalog === "probe") assert.strictEqual(failed?.version, null);
205+
assert.strictEqual(discoveryLogs.length, 1);
206+
assert.strictEqual(discoveryLogs[0]?.providerInstanceId, "opencode-test");
207+
assert.strictEqual(discoveryLogs[0]?.serverMode, "external");
208+
assert.strictEqual(
209+
discoveryLogs[0]?.phase,
210+
initialCatalog === "probe" ? "version" : "models",
211+
);
212+
213+
catalogAvailable = true;
214+
const recovered = yield* instance.snapshot.streamChanges.pipe(
215+
Stream.filter((snapshot) => snapshot.status === "ready"),
216+
Stream.take(1),
217+
Stream.runCollect,
218+
Effect.forkChild,
219+
);
220+
yield* TestClock.adjust("5 minutes");
221+
assert.strictEqual((yield* instance.snapshot.getSnapshot).status, "ready");
222+
const [ready] = Array.from(yield* Fiber.join(recovered));
223+
assert.strictEqual(ready?.auth.status, "authenticated");
224+
assert.deepStrictEqual(
225+
ready?.models.map((model) => model.slug),
226+
["opencode/big-pickle"],
227+
);
228+
229+
const readsAfterRecovery = modelReads;
230+
yield* TestClock.adjust("15 minutes");
231+
assert.strictEqual(modelReads, readsAfterRecovery);
232+
assert.strictEqual(discoveryLogs.length, 1);
233+
}).pipe(Effect.scoped, Effect.provide(TestClock.layer())),
234+
);
235+
236+
it.effect("does not retry a rejected server password in the background", () =>
237+
Effect.gen(function* () {
238+
const releaseResponse = yield* Deferred.make<void>();
239+
const requested: Array<string> = [];
240+
const server = replayOpenCodeServer(OPENCODE_2_RESPONSES, "secret", requested);
241+
const instance = yield* create(
242+
{ serverUrl: "http://127.0.0.1:4096", serverPassword: "wrong" },
243+
HttpClient.make((request) =>
244+
Deferred.await(releaseResponse).pipe(Effect.andThen(server.execute(request))),
245+
),
246+
);
247+
// Wait for startup and its maintenance enrichment, both of which probe.
248+
const rejected = yield* instance.snapshot.streamChanges.pipe(
249+
Stream.filter(
250+
(snapshot) => snapshot.status === "error" && snapshot.versionAdvisory !== undefined,
251+
),
252+
Stream.take(1),
253+
Stream.runCollect,
254+
Effect.forkChild,
255+
);
256+
yield* Effect.yieldNow;
257+
yield* Deferred.succeed(releaseResponse, undefined);
258+
const [snapshot] = Array.from(yield* Fiber.join(rejected));
259+
assert.strictEqual(snapshot?.version, null);
260+
assert.strictEqual(snapshot?.auth.status, "unauthenticated");
261+
const readsAfterRejection = requested.length;
262+
263+
yield* TestClock.adjust("15 minutes");
264+
assert.strictEqual(requested.length, readsAfterRejection);
265+
}).pipe(Effect.scoped, Effect.provide(TestClock.layer())),
266+
);
97267
});
98268

99269
// npm links `<prefix>/bin/opencode` into the package that installed it:

‎packages/provider-opencode/src/server/driver.ts‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -350,6 +350,9 @@ export const OpenCodeDriver: ProviderDriver<
350350
host.paths.cwd,
351351
runtimeProbe.refresh,
352352
loadOpenCode2Models,
353+
).pipe(
354+
Effect.annotateLogs({ providerInstanceId: instanceId }),
355+
Effect.annotateSpans({ providerInstanceId: instanceId }),
353356
),
354357
usageLimits: readOpenCodeGoUsageLimits({
355358
enabled: effectiveConfig.enabled,
@@ -440,7 +443,13 @@ export const OpenCodeDriver: ProviderDriver<
440443
streamSettings: snapshotSettings.streamSettings,
441444
haveSettingsChanged: haveProviderSnapshotSettingsChanged,
442445
checkProviderOnSettingsChange: () => false,
443-
refreshOnInterval: false,
446+
// A failed or still-loading catalog must recover without a manual
447+
// refresh. Healthy instances need no recurring server starts.
448+
refreshOnInterval: (snapshot) =>
449+
snapshot.enabled &&
450+
snapshot.installed &&
451+
snapshot.status !== "ready" &&
452+
snapshot.auth.status === "unknown",
444453
initialSnapshot: (settings) =>
445454
makePendingOpenCodeProvider(settings.provider).pipe(Effect.map(stampIdentity)),
446455
checkProvider,

0 commit comments

Comments
 (0)