Skip to content

Commit d47f2e0

Browse files
fix(server): providers keep their saved status until the first check finishes (pingdotgg#34678)
1 parent 7d71f85 commit d47f2e0

5 files changed

Lines changed: 144 additions & 15 deletions

File tree

‎apps/server/src/provider/ProviderRegistry.test.ts‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1270,6 +1270,9 @@ it.layer(
12701270
}),
12711271
),
12721272
getSnapshot: Ref.get(currentProvider),
1273+
isAwaitingFirstCheck: Ref.get(currentProvider).pipe(
1274+
Effect.map((provider) => provider === pendingProvider),
1275+
),
12731276
refresh: Ref.get(nextProvider).pipe(
12741277
Effect.tap((provider) => Ref.set(currentProvider, provider)),
12751278
),
@@ -1303,6 +1306,23 @@ it.layer(
13031306
? retainedModels
13041307
: [customModel, ...cachedProvider.models];
13051308
assert.deepStrictEqual((yield* registry.getProviders)[0]?.models, expectedModels);
1309+
if (!restarted) {
1310+
const probeFields = ({ status, installed, auth, version }: ServerProvider) => ({
1311+
status,
1312+
installed,
1313+
auth,
1314+
version,
1315+
});
1316+
// Boot reads the probe state from the cache, not the pending snapshot, and never persists the pending one.
1317+
assert.deepStrictEqual(
1318+
probeFields((yield* registry.getProviders)[0]!),
1319+
probeFields(cachedProvider),
1320+
);
1321+
assert.deepStrictEqual(
1322+
probeFields((yield* readProviderStatusCache(filePath))!),
1323+
probeFields(cachedProvider),
1324+
);
1325+
}
13061326

13071327
yield* registry.refreshInstance(instance.instanceId);
13081328
assert.deepStrictEqual(
@@ -1335,6 +1355,84 @@ it.layer(
13351355
),
13361356
),
13371357
);
1358+
it.effect("keeps a check that finished before boot over the cached status", () =>
1359+
Effect.gen(function* () {
1360+
const config = yield* ServerConfig.ServerConfig;
1361+
const filePath = yield* resolveProviderStatusCachePath({
1362+
cacheDir: config.providerStatusCacheDir,
1363+
instanceId: cachedProvider.instanceId,
1364+
});
1365+
const instance = {
1366+
instanceId: cachedProvider.instanceId,
1367+
driverKind: cachedProvider.driver,
1368+
continuationIdentity: {
1369+
driverKind: cachedProvider.driver,
1370+
continuationKey: "codex:instance:codex-personal",
1371+
},
1372+
displayName: undefined,
1373+
enabled: true,
1374+
snapshot: {
1375+
resolveMaintenance: () =>
1376+
Effect.succeed(
1377+
makeManualOnlyProviderMaintenanceCapabilities({
1378+
provider: cachedProvider.driver,
1379+
packageName: null,
1380+
}),
1381+
),
1382+
getSnapshot: Effect.succeed(failedProvider),
1383+
isAwaitingFirstCheck: Effect.succeed(false),
1384+
refresh: Effect.succeed(failedProvider),
1385+
streamChanges: Stream.empty,
1386+
applyUsageLimits: () => Effect.void,
1387+
},
1388+
orchestrationAdapter: {} as ProviderInstance["orchestrationAdapter"],
1389+
textGeneration: {} as ProviderInstance["textGeneration"],
1390+
} satisfies ProviderInstance;
1391+
const layerInstanceRegistry = Layer.succeed(
1392+
ProviderInstanceRegistry.ProviderInstanceRegistry,
1393+
{
1394+
getInstance: (id) =>
1395+
Effect.succeed(id === instance.instanceId ? instance : undefined),
1396+
listInstances: Effect.succeed([instance]),
1397+
listUnavailable: Effect.succeed([]),
1398+
streamChanges: Stream.empty,
1399+
subscribeChanges: Effect.flatMap(PubSub.unbounded<void>(), PubSub.subscribe),
1400+
},
1401+
);
1402+
1403+
// First boot has no cache file; the second has a stale ready one.
1404+
for (const hasCache of [false, true]) {
1405+
if (hasCache) {
1406+
yield* writeProviderStatusCache({
1407+
filePath,
1408+
provider: { ...cachedProvider, status: "ready" },
1409+
});
1410+
}
1411+
yield* Effect.gen(function* () {
1412+
const registry = yield* ProviderRegistry.ProviderRegistry;
1413+
assert.strictEqual((yield* registry.getProviders)[0]?.status, failedProvider.status);
1414+
assert.strictEqual(
1415+
(yield* readProviderStatusCache(filePath))?.status,
1416+
failedProvider.status,
1417+
);
1418+
}).pipe(
1419+
Effect.provide(
1420+
ProviderRegistry.layer.pipe(
1421+
Layer.provide(layerInstanceRegistry),
1422+
Layer.provide(layerUnusedMutator),
1423+
),
1424+
),
1425+
Effect.scoped,
1426+
);
1427+
}
1428+
}).pipe(
1429+
Effect.provide(
1430+
ServerConfig.layerTest(process.cwd(), {
1431+
prefix: "t3-codex-probed-before-boot-",
1432+
}).pipe(Layer.provideMerge(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))),
1433+
),
1434+
),
1435+
);
13381436
});
13391437

13401438
describe("Antigravity model inventories", () => {

‎apps/server/src/provider/ProviderRegistry.ts‎

Lines changed: 38 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -386,10 +386,17 @@ export const layer = Layer.effect(
386386
}
387387
fallbackByInstance.set(source.instanceId, provider);
388388
}
389+
// Read after the snapshots: a snapshot read as awaiting is the pre-check placeholder.
390+
const awaitingBootInstances = new Set<ProviderInstance>();
391+
for (const instance of bootInstances) {
392+
if (yield* instance.snapshot.isAwaitingFirstCheck ?? Effect.succeed(false)) {
393+
awaitingBootInstances.add(instance);
394+
}
395+
}
389396

390-
const cachedProviders = yield* Effect.forEach(
397+
const bootProviders = yield* Effect.forEach(
391398
bootSources,
392-
(source) =>
399+
(source, index) =>
393400
Effect.gen(function* () {
394401
const filePath = yield* resolveProviderStatusCachePath({
395402
cacheDir: config.providerStatusCacheDir,
@@ -399,11 +406,15 @@ export const layer = Layer.effect(
399406
if (fallbackProvider === undefined) {
400407
return undefined;
401408
}
409+
// A completed check is seeded by the boot upsert instead, which persists it.
410+
const placeholder = awaitingBootInstances.has(bootInstances[index]!)
411+
? fallbackProvider
412+
: undefined;
402413
return yield* readProviderStatusCache(filePath).pipe(
403414
Effect.provideService(FileSystem.FileSystem, fileSystem),
404415
Effect.flatMap((cachedProvider) => {
405416
if (cachedProvider === undefined) {
406-
return Effect.void.pipe(Effect.as(undefined as ServerProvider | undefined));
417+
return Effect.succeed(placeholder);
407418
}
408419
const correlation = {
409420
cachedProvider,
@@ -416,7 +427,7 @@ export const layer = Layer.effect(
416427
cachedInstanceId: cachedProvider.instanceId ?? null,
417428
driver: source.driverKind,
418429
cachedDriver: cachedProvider.driver ?? null,
419-
}).pipe(Effect.as(undefined as ServerProvider | undefined));
430+
}).pipe(Effect.as(placeholder));
420431
}
421432
return Effect.succeed(hydrateCachedProvider(correlation));
422433
}),
@@ -441,7 +452,7 @@ export const layer = Layer.effect(
441452
ModelManifest.BUNDLED_MODEL_MANIFEST.compatibility,
442453
);
443454
const providersRef = yield* Ref.make<ReadonlyArray<ServerProvider>>(
444-
cachedProviders.map((provider) => classifyCompatibility(provider, initialManifest)),
455+
bootProviders.map((provider) => classifyCompatibility(provider, initialManifest)),
445456
);
446457
const workspaceRefreshesRef = yield* Ref.make<
447458
ReadonlyMap<ProviderInstance, ReadonlyMap<string, Deferred.Deferred<void>>>
@@ -551,21 +562,24 @@ export const layer = Layer.effect(
551562
});
552563

553564
const compatibilityRefreshRunning = yield* Ref.make(false);
554-
const syncProvider = Effect.fn("syncProvider")(function* (
555-
provider: ServerProvider,
556-
options?: {
557-
readonly publish?: boolean;
558-
},
559-
) {
560-
const providers = yield* upsertProviders([provider], options);
561-
// Reclassify the current model after fetch; republishing the old probe could overwrite newer health results.
565+
// Reclassify the current model after fetch; republishing the old probe could overwrite newer health results.
566+
const refreshCompatibility = Effect.gen(function* () {
562567
if (!(yield* Ref.getAndSet(compatibilityRefreshRunning, true))) {
563568
yield* manifestService.refresh.pipe(
564569
Effect.andThen(upsertProviders([], { persist: false })),
565570
Effect.ensuring(Ref.set(compatibilityRefreshRunning, false)),
566571
Effect.forkIn(serviceScope),
567572
);
568573
}
574+
});
575+
const syncProvider = Effect.fn("syncProvider")(function* (
576+
provider: ServerProvider,
577+
options?: {
578+
readonly publish?: boolean;
579+
},
580+
) {
581+
const providers = yield* upsertProviders([provider], options);
582+
yield* refreshCompatibility;
569583
return providers;
570584
});
571585

@@ -861,6 +875,13 @@ export const layer = Layer.effect(
861875
Effect.gen(function* () {
862876
const source = buildSnapshotSource(instance);
863877
const provider = yield* source.getSnapshot;
878+
// Boot shows the cached status until this instance's first check publishes a result.
879+
if (
880+
awaitingBootInstances.has(instance) &&
881+
(yield* instance.snapshot.isAwaitingFirstCheck ?? Effect.succeed(false))
882+
) {
883+
return yield* refreshCompatibility;
884+
}
864885
yield* correlateSnapshotWithSource(source, provider).pipe(
865886
Effect.flatMap(syncProvider),
866887
);
@@ -918,7 +939,10 @@ export const layer = Layer.effect(
918939
Scope.provide(serviceScope),
919940
);
920941

921-
yield* upsertProviders(fallbackProviders, { publish: false });
942+
yield* upsertProviders(
943+
fallbackProviders.filter((_, index) => !awaitingBootInstances.has(bootInstances[index]!)),
944+
{ publish: false },
945+
);
922946
const instanceChanges = yield* instanceRegistry.subscribeChanges;
923947
yield* syncLiveSources;
924948
yield* Stream.runForEach(

‎packages/provider-core/src/server/managedProvider.test.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -272,6 +272,7 @@ describe("makeManagedServerProvider", () => {
272272

273273
const initial = yield* provider.getSnapshot;
274274
assert.deepStrictEqual(initial, initialSnapshot);
275+
assert.strictEqual(yield* provider.isAwaitingFirstCheck!, true);
275276

276277
const updatesFiber = yield* Stream.take(provider.streamChanges, 1).pipe(
277278
Stream.runCollect,
@@ -287,6 +288,7 @@ describe("makeManagedServerProvider", () => {
287288

288289
assert.deepStrictEqual(updates, [refreshedSnapshot]);
289290
assert.deepStrictEqual(latest, refreshedSnapshot);
291+
assert.strictEqual(yield* provider.isAwaitingFirstCheck!, false);
290292
assert.strictEqual(yield* Ref.get(checkCalls), 1);
291293
}),
292294
).pipe(Effect.provide(layerAlwaysRunTest)),

‎packages/provider-core/src/server/managedProvider.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ interface ProviderSnapshotState {
2828
readonly enrichmentGeneration: number;
2929
// null preserves an explicitly cleared plan across probes and enrichment.
3030
readonly usageAuthLabel: string | null | undefined;
31+
readonly checked: boolean;
3132
}
3233

3334
function withUsageLimits(
@@ -79,6 +80,7 @@ export const makeManagedServerProvider = Effect.fn("makeManagedServerProvider")(
7980
snapshot: initialSnapshot,
8081
enrichmentGeneration: 0,
8182
usageAuthLabel: undefined,
83+
checked: false,
8284
});
8385
const settingsRef = yield* Ref.make(initialSettings);
8486
const enrichmentFiberRef = yield* Ref.make<Fiber.Fiber<void, unknown> | null>(null);
@@ -218,7 +220,7 @@ export const makeManagedServerProvider = Effect.fn("makeManagedServerProvider")(
218220
);
219221
return [
220222
{ snapshot, generation },
221-
{ snapshot, enrichmentGeneration: generation, usageAuthLabel },
223+
{ snapshot, enrichmentGeneration: generation, usageAuthLabel, checked: true },
222224
] as const;
223225
},
224226
);
@@ -425,6 +427,7 @@ export const makeManagedServerProvider = Effect.fn("makeManagedServerProvider")(
425427
return {
426428
resolveMaintenance: input.resolveMaintenance,
427429
getSnapshot: Ref.get(snapshotStateRef).pipe(Effect.map((state) => state.snapshot)),
430+
isAwaitingFirstCheck: Ref.get(snapshotStateRef).pipe(Effect.map((state) => !state.checked)),
428431
refresh: input
429432
.resolveMaintenance({ fresh: true })
430433
.pipe(Effect.andThen(refreshSnapshot), Effect.tapError(Effect.logError), Effect.orDie),

‎packages/provider-core/src/server/snapshot.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@ export interface ManagedServerProvider {
2121
readonly fresh?: boolean;
2222
}) => Effect.Effect<ProviderMaintenanceCapabilities>;
2323
readonly getSnapshot: Effect.Effect<ServerProvider>;
24+
/** True while getSnapshot still returns the placeholder from before the first completed check. */
25+
readonly isAwaitingFirstCheck?: Effect.Effect<boolean>;
2426
readonly refresh: Effect.Effect<ServerProvider>;
2527
readonly refreshUsage?: Effect.Effect<void, ServerSettingsError>;
2628
readonly streamChanges: Stream.Stream<ServerProvider>;

0 commit comments

Comments
 (0)