Skip to content

Commit c8de687

Browse files
yaacovcorcosclaude
andcommitted
fix(server): provider settings saved during startup are no longer lost
The provider registry read its seed settings first and subscribed to settings changes only after every provider instance was built. A change saved in between was published to no one, so the registry kept the old providers until the next unrelated settings change. Acquire the settings subscription before reading the seed and hand it to the watcher, so a change made while the registry builds is queued and applied once the watcher starts. Because a queued change can now be older than the seed, the watcher reconciles against the current settings on each change instead of the queued snapshot. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1 parent 9bd1d80 commit c8de687

2 files changed

Lines changed: 191 additions & 22 deletions

File tree

‎apps/server/src/provider/ProviderInstanceRegistryHydration.ts‎

Lines changed: 33 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -28,11 +28,14 @@
2828
* Hot-reload
2929
* ----------
3030
* On layer build we:
31-
* 1. Read the current `ServerSettings` once and use it to seed the
31+
* 1. Acquire `ServerSettingsService.subscribeChanges`, so a change saved
32+
* while the registry is still building is queued rather than lost.
33+
* 2. Read the current `ServerSettings` once and use it to seed the
3234
* registry's initial state via `ProviderInstanceRegistry.layer`.
33-
* 2. Fork a daemon fiber (lifetime tied to the layer's scope) that
34-
* acquires `ServerSettingsService.subscribeChanges` and calls
35-
* `ProviderInstanceRegistryMutator.reconcile` on every emission.
35+
* 3. Fork a daemon fiber (lifetime tied to the layer's scope) that, on
36+
* every emission, re-reads the current settings and calls
37+
* `ProviderInstanceRegistryMutator.reconcile` with them. A queued
38+
* change may predate the seed, so its snapshot is not applied as is.
3639
*
3740
* Failures inside the watcher are logged and swallowed so a single bad
3841
* settings emission cannot kill the registry. Unknown drivers and invalid
@@ -124,25 +127,29 @@ export const deriveProviderInstanceConfigMap = (
124127
* configs, so the only way the watcher could fail is a settings stream
125128
* tear-down, which logs and exits cleanly.
126129
*/
127-
const layerSettingsWatcher = Layer.effectDiscard(
128-
Effect.gen(function* () {
129-
const mutator = yield* ProviderInstanceRegistryMutator.ProviderInstanceRegistryMutator;
130-
const serverSettings = yield* Settings.ServerSettingsService;
131-
const settingsChanges = yield* serverSettings.subscribeChanges;
132-
yield* settingsChanges.pipe(
133-
Stream.runForEach((next) =>
134-
mutator
135-
.reconcile(deriveProviderInstanceConfigMap(next))
136-
.pipe(
130+
const layerSettingsWatcher = (settingsChanges: Stream.Stream<ServerSettings>) =>
131+
Layer.effectDiscard(
132+
Effect.gen(function* () {
133+
const mutator = yield* ProviderInstanceRegistryMutator.ProviderInstanceRegistryMutator;
134+
const serverSettings = yield* Settings.ServerSettingsService;
135+
yield* settingsChanges.pipe(
136+
Stream.runForEach((next) =>
137+
// The subscription predates the seed read, so a queued change can be
138+
// older than the seed. Reconcile against the current settings.
139+
serverSettings.getSettings.pipe(
140+
Effect.orElseSucceed(() => next),
141+
Effect.flatMap((current) =>
142+
mutator.reconcile(deriveProviderInstanceConfigMap(current)),
143+
),
137144
Effect.catchCause((cause) =>
138145
Effect.logError("ProviderInstanceRegistry reconcile failed", cause),
139146
),
140147
),
141-
),
142-
Effect.forkScoped,
143-
);
144-
}),
145-
);
148+
),
149+
Effect.forkScoped,
150+
);
151+
}),
152+
);
146153

147154
/**
148155
* Hydrate `ProviderInstanceRegistry` from `ServerSettings` and keep it in
@@ -152,8 +159,9 @@ const layerSettingsWatcher = Layer.effectDiscard(
152159
* - `ProviderInstanceRegistry.layer` produces the registry +
153160
* mutator from the initial config map. Its scope owns every
154161
* per-instance child scope created during reconcile.
155-
* - `SettingsWatcherLive` consumes the mutator, acquires its settings
156-
* subscription before forking, and runs a daemon fiber in the same scope.
162+
* - `layerSettingsWatcher` consumes the mutator and drains the settings
163+
* subscription, acquired before the seed read, in a daemon fiber in the
164+
* same scope.
157165
*
158166
* Composing via `Layer.provideMerge` makes the watcher's deps available
159167
* from the mutable layer while still surfacing the registry as an output.
@@ -167,6 +175,9 @@ export const layer: Layer.Layer<
167175
> = Layer.unwrap(
168176
Effect.gen(function* () {
169177
const serverSettings = yield* Settings.ServerSettingsService;
178+
// Subscribe before reading the seed: building the registry takes a while,
179+
// and a change published before the watcher subscribed would never arrive.
180+
const settingsChanges = yield* serverSettings.subscribeChanges;
170181
const initialSettings: ServerSettings | undefined = yield* serverSettings.getSettings.pipe(
171182
Effect.orElseSucceed(() => undefined),
172183
);
@@ -183,7 +194,7 @@ export const layer: Layer.Layer<
183194
Layer.provide(AcpRegistryCatalog.layer),
184195
);
185196

186-
return layerSettingsWatcher.pipe(Layer.provideMerge(layerMutable));
197+
return layerSettingsWatcher(settingsChanges).pipe(Layer.provideMerge(layerMutable));
187198
}),
188199
) as Layer.Layer<
189200
ProviderInstanceRegistry.ProviderInstanceRegistry,

‎apps/server/src/provider/ProviderRegistry.test.ts‎

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2861,6 +2861,164 @@ it.layer(
28612861
}),
28622862
);
28632863

2864+
// Hydrates the provider registry. `onSeedRead` runs right after the
2865+
// registry reads its seed settings, before the rest of it has finished
2866+
// building. Returns once every settings consumer has caught up.
2867+
const startupSettings = decodeServerSettings(
2868+
deepMerge(encodedDefaultServerSettings, {
2869+
providers: {
2870+
codex: { enabled: false },
2871+
claudeAgent: { enabled: false },
2872+
cursor: { enabled: false },
2873+
grok: { enabled: false },
2874+
opencode: { enabled: false },
2875+
},
2876+
}),
2877+
);
2878+
const hydrateWithSeedReadHook = (
2879+
seed: ContractServerSettings,
2880+
onSeedRead: (input: {
2881+
readonly settingsRef: Ref.Ref<ContractServerSettings>;
2882+
readonly changes: PubSub.PubSub<ContractServerSettings>;
2883+
}) => Effect.Effect<void>,
2884+
) =>
2885+
Effect.gen(function* () {
2886+
const settingsRef = yield* Ref.make(seed);
2887+
const changes = yield* PubSub.unbounded<ContractServerSettings>();
2888+
let saveDuringStartup: Effect.Effect<void> | undefined;
2889+
// Every settings consumer reports when it has handled everything
2890+
// published to it and is waiting for more.
2891+
const consumersIdle = new Map<number, boolean>();
2892+
const allConsumersCaughtUp = yield* Deferred.make<void>();
2893+
const setIdle = (consumer: number, idle: boolean) =>
2894+
Effect.suspend(() => {
2895+
consumersIdle.set(consumer, idle);
2896+
return [...consumersIdle.values()].every(Boolean)
2897+
? Deferred.succeed(allConsumersCaughtUp, undefined)
2898+
: Effect.void;
2899+
});
2900+
const serverSettings = {
2901+
start: Effect.void,
2902+
ready: Effect.void,
2903+
getSettings: Effect.suspend(() => {
2904+
const save = saveDuringStartup;
2905+
saveDuringStartup = undefined;
2906+
return Ref.get(settingsRef).pipe(Effect.tap(() => save ?? Effect.void));
2907+
}),
2908+
updateSettings: () => Effect.die("unused"),
2909+
updateProviderInstance: () => Effect.die("unused"),
2910+
withSettingsSnapshot: (use) => Ref.get(settingsRef).pipe(Effect.flatMap(use)),
2911+
streamChanges: Stream.fromPubSub(changes),
2912+
subscribeChanges: Effect.gen(function* () {
2913+
const subscription = yield* PubSub.subscribe(changes);
2914+
const consumer = consumersIdle.size;
2915+
consumersIdle.set(consumer, false);
2916+
return Stream.fromEffectRepeat(
2917+
Effect.gen(function* () {
2918+
if ((yield* PubSub.remaining(subscription)) === 0) {
2919+
yield* setIdle(consumer, true);
2920+
}
2921+
const next = yield* PubSub.take(subscription);
2922+
yield* setIdle(consumer, false);
2923+
return next;
2924+
}),
2925+
);
2926+
}),
2927+
} satisfies ServerSettingsModule.ServerSettingsService["Service"];
2928+
const scope = yield* Scope.make();
2929+
yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void));
2930+
const dependencies = yield* Layer.build(
2931+
Layer.mergeAll(
2932+
AntigravityInstallation.AntigravityInstallation.layer,
2933+
ServerSecretStore.layer,
2934+
OpenCodeRuntime.layer.pipe(Layer.provide(OpenCodeServerLedger.layerTest)),
2935+
).pipe(
2936+
Layer.provideMerge(
2937+
Layer.succeed(ServerSettingsModule.ServerSettingsService, serverSettings),
2938+
),
2939+
Layer.provideMerge(
2940+
ServerConfig.layerTest(process.cwd(), {
2941+
prefix: "t3-provider-registry-",
2942+
}),
2943+
),
2944+
Layer.provideMerge(layerTestHttpClient),
2945+
Layer.provideMerge(
2946+
Layer.succeed(
2947+
ProviderEventLoggers.ProviderEventLoggers,
2948+
ProviderEventLoggers.NoOpProviderEventLoggers,
2949+
),
2950+
),
2951+
Layer.provideMerge(ModelManifest.layerTest),
2952+
Layer.provideMerge(ResetCreditCoordinator.layerTest),
2953+
Layer.provideMerge(NodeServices.layer),
2954+
Layer.provideMerge(layerBackgroundPolicyAlwaysRun),
2955+
),
2956+
).pipe(Scope.provide(scope));
2957+
2958+
saveDuringStartup = onSeedRead({ settingsRef, changes });
2959+
const registryServices = yield* Layer.build(ProviderInstanceRegistryHydration.layer).pipe(
2960+
Effect.provide(dependencies),
2961+
Scope.provide(scope),
2962+
);
2963+
yield* Deferred.await(allConsumersCaughtUp);
2964+
2965+
const registry = yield* ProviderInstanceRegistry.ProviderInstanceRegistry.pipe(
2966+
Effect.provide(registryServices),
2967+
);
2968+
return (yield* registry.listUnavailable).map(
2969+
(provider): string | undefined => provider.instanceId,
2970+
);
2971+
});
2972+
const withGhostInstance = (settings: ContractServerSettings, instanceId: string) =>
2973+
({
2974+
...settings,
2975+
providerInstances: {
2976+
...settings.providerInstances,
2977+
[instanceId]: {
2978+
driver: "ghostDriver",
2979+
displayName: "Added while the registry started",
2980+
enabled: false,
2981+
config: {},
2982+
},
2983+
},
2984+
}) as unknown as ContractServerSettings;
2985+
2986+
it.effect("applies a settings change saved while the provider registry starts", () =>
2987+
Effect.gen(function* () {
2988+
const unavailable = yield* hydrateWithSeedReadHook(
2989+
startupSettings,
2990+
({ settingsRef, changes }) =>
2991+
Ref.updateAndGet(settingsRef, (current) =>
2992+
withGhostInstance(current, "ghost_late"),
2993+
).pipe(
2994+
Effect.flatMap((next) => PubSub.publish(changes, next)),
2995+
Effect.asVoid,
2996+
),
2997+
);
2998+
assert.include(
2999+
unavailable,
3000+
"ghost_late",
3001+
"the instance saved during startup is missing from the registry",
3002+
);
3003+
}),
3004+
);
3005+
3006+
it.effect("does not apply a queued settings change older than the registry seed", () =>
3007+
Effect.gen(function* () {
3008+
// The seed already has `ghost_current`; a change published before the
3009+
// seed read is still queued and carries older settings.
3010+
const unavailable = yield* hydrateWithSeedReadHook(
3011+
withGhostInstance(startupSettings, "ghost_current"),
3012+
({ changes }) =>
3013+
PubSub.publish(changes, withGhostInstance(startupSettings, "ghost_stale")).pipe(
3014+
Effect.asVoid,
3015+
),
3016+
);
3017+
assert.include(unavailable, "ghost_current");
3018+
assert.notInclude(unavailable, "ghost_stale");
3019+
}),
3020+
);
3021+
28643022
it.effect(
28653023
"keeps Cursor disabled and skips provider probing when settings use their defaults",
28663024
() =>

0 commit comments

Comments
 (0)