Skip to content

Commit b1b3c10

Browse files
only21milcodexmaria-rcks
authored
fix(clients): keep accepted server updates through disconnects (#23532)
Co-authored-by: GPT-6.1 Sol <[email protected]> Co-authored-by: maria-rcks <[email protected]>
1 parent 269357c commit b1b3c10

4 files changed

Lines changed: 313 additions & 4 deletions

File tree

‎packages/client-runtime/src/connection/outdatedHostUpdate.test.ts‎

Lines changed: 123 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,20 @@ class OutdatedHostSocket {
124124
const message = decodeJson(data);
125125
if (!isRpcRequest(message)) return;
126126
this.requests.push(message);
127+
if (message.tag === WS_METHODS.serverUpdateServerWithProgress) {
128+
this.onUpdate();
129+
queueMicrotask(() =>
130+
this.emit({
131+
type: "message",
132+
data: encodeJson({
133+
_tag: "Chunk",
134+
requestId: message.id,
135+
values: [{ type: "progress", stage: "downloading" }],
136+
}),
137+
}),
138+
);
139+
return;
140+
}
127141
if (message.tag !== WS_METHODS.serverUpdateServer) return;
128142
this.onUpdate();
129143
queueMicrotask(() =>
@@ -143,10 +157,10 @@ class OutdatedHostSocket {
143157

144158
close() {
145159
this.readyState = 3;
146-
this.emit({ type: "close" });
160+
this.emit({ type: "close", code: 1006 });
147161
}
148162

149-
private emit(event: { readonly type: string; readonly data?: unknown }) {
163+
private emit(event: { readonly type: string; readonly data?: unknown; readonly code?: number }) {
150164
for (const listener of this.listeners.get(event.type) ?? []) listener(event);
151165
}
152166
}
@@ -314,4 +328,111 @@ describe("updateOutdatedHost", () => {
314328
expect(opened).toBe(false);
315329
}),
316330
);
331+
332+
it.effect("resumes an outdated host update after progress transport loss", () =>
333+
Effect.gen(function* () {
334+
const blocked = orchestrationProtocolCompatibilityError(descriptor(undefined, "0.0.45"));
335+
expect(blocked).toMatchObject({ serverUpdateRequired: true });
336+
337+
const sockets: Array<OutdatedHostSocket> = [];
338+
let served: ExecutionEnvironmentDescriptor = {
339+
...descriptor(undefined, "0.0.45"),
340+
capabilities: {
341+
repositoryIdentity: true,
342+
serverSelfUpdate: "boot-service",
343+
serverSelfUpdateProgress: true,
344+
},
345+
};
346+
const entries = yield* SubscriptionRef.make<
347+
ReadonlyMap<EnvironmentId, ConnectionCatalogEntry>
348+
>(
349+
new Map([
350+
[
351+
TARGET.environmentId,
352+
{
353+
target: TARGET,
354+
profile: Option.none(),
355+
enabled: false,
356+
unsupportedReason: blocked?.message ?? "",
357+
serverUpdateRequired: true,
358+
},
359+
],
360+
]),
361+
);
362+
const calls: Array<string> = [];
363+
const registry = EnvironmentRegistry.EnvironmentRegistry.of({
364+
entries,
365+
setCompatibility: (_environmentId: EnvironmentId, error: unknown) =>
366+
Effect.sync(() => calls.push(`compatibility:${error === null ? "clear" : "block"}`)),
367+
setEnabled: (_environmentId: EnvironmentId, enabled: boolean) =>
368+
Effect.sync(() => calls.push(`enabled:${enabled}`)),
369+
} as unknown as EnvironmentRegistry.EnvironmentRegistry["Service"]);
370+
const resolver = ConnectionResolver.ConnectionResolver.of({
371+
prepare: () => Effect.die(new Error("The update must bypass the protocol gate.")),
372+
prepareForUpdate: () =>
373+
Effect.sync(() => served).pipe(
374+
Effect.map((current) => ({
375+
descriptor: current,
376+
prepared: {
377+
environmentId: TARGET.environmentId,
378+
label: TARGET.label,
379+
httpBaseUrl: TARGET.httpBaseUrl,
380+
socketUrl: "wss://build.example.test/ws?wsTicket=ticket",
381+
httpAuthorization: null,
382+
target: TARGET,
383+
},
384+
})),
385+
),
386+
});
387+
const httpClient = HttpClient.make((request) =>
388+
Effect.sync(() =>
389+
HttpClientResponse.fromWeb(request, new Response(encodeDescriptor(served))),
390+
),
391+
);
392+
393+
const stages: Array<string> = [];
394+
const result = yield* updateOutdatedHost(
395+
TARGET.environmentId,
396+
{ targetVersion: "0.0.46" },
397+
(stage) =>
398+
Effect.sync(() => {
399+
stages.push(stage);
400+
// Lose the transport only after the consumer has observed progress.
401+
if (stage === "downloading") sockets[0]?.close();
402+
}),
403+
).pipe(
404+
Effect.provide(
405+
Layer.mergeAll(
406+
Layer.succeed(EnvironmentRegistry.EnvironmentRegistry, registry),
407+
Layer.succeed(ConnectionResolver.ConnectionResolver, resolver),
408+
Layer.succeed(
409+
RelayEnvironmentDiscovery.RelayEnvironmentDiscovery,
410+
RelayEnvironmentDiscovery.RelayEnvironmentDiscovery.of({
411+
state: yield* SubscriptionRef.make(
412+
RelayEnvironmentDiscovery.EMPTY_RELAY_ENVIRONMENT_DISCOVERY_STATE,
413+
),
414+
refresh: Effect.void,
415+
}),
416+
),
417+
Layer.succeed(HttpClient.HttpClient, httpClient),
418+
Layer.succeed(Socket.WebSocketConstructor, (url) => {
419+
const socket = new OutdatedHostSocket(url, () => {
420+
served = descriptor(ORCHESTRATION_PROTOCOL_VERSION, "0.0.46");
421+
});
422+
sockets.push(socket);
423+
return socket as unknown as globalThis.WebSocket;
424+
}),
425+
),
426+
),
427+
);
428+
429+
expect(sockets[0]?.url).not.toContain("orchestrationProtocol");
430+
expect(sockets[0]?.requests.map((request) => request.tag)).toEqual([
431+
WS_METHODS.serverUpdateServerWithProgress,
432+
]);
433+
expect(stages).toEqual(["downloading", "resuming"]);
434+
expect(result.targetVersion).toBe("0.0.46");
435+
expect(calls).toEqual(["compatibility:clear", "enabled:true"]);
436+
}),
437+
);
317438
});

‎packages/client-runtime/src/connection/outdatedHostUpdate.ts‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -141,20 +141,31 @@ export const updateOutdatedHost = Effect.fn("clientRuntime.connection.updateOutd
141141
capabilities.serverSelfUpdateProgress === true
142142
? yield* Effect.gen(function* () {
143143
const terminal = yield* Ref.make(Option.none<ServerSelfUpdateResult>());
144+
let receivedProgress = false;
144145
const streamExit = yield* client[WS_METHODS.serverUpdateServerWithProgress](
145146
input,
146147
).pipe(
147148
Stream.runForEach((event) =>
148149
event.type === "complete"
149150
? Ref.set(terminal, Option.some(event.result))
150-
: onStage(event.stage),
151+
: Effect.sync(() => {
152+
receivedProgress = true;
153+
}).pipe(Effect.andThen(onStage(event.stage))),
151154
),
152155
Effect.exit,
153156
);
157+
const selfUpdateMethod = capabilities.serverSelfUpdate;
158+
// Desktop preparation needs its terminal commit token.
159+
const accepted =
160+
receivedProgress &&
161+
(selfUpdateMethod === "boot-service" || selfUpdateMethod === "respawn")
162+
? { targetVersion: input.targetVersion, method: selfUpdateMethod }
163+
: undefined;
154164
return yield* resolveServerUpdateProgressResult(
155165
input.targetVersion,
156166
yield* Ref.get(terminal),
157167
streamExit,
168+
accepted,
158169
);
159170
})
160171
: yield* client[WS_METHODS.serverUpdateServer](input).pipe(

‎packages/client-runtime/src/state/server.test.ts‎

Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import {
22
EnvironmentId,
3+
ServerSelfUpdateError,
34
ProviderInstanceId,
45
ServerObservability,
56
EnvironmentThemeId,
@@ -16,12 +17,14 @@ import * as Duration from "effect/Duration";
1617
import * as Effect from "effect/Effect";
1718
import * as Exit from "effect/Exit";
1819
import * as Fiber from "effect/Fiber";
20+
import * as Layer from "effect/Layer";
1921
import * as Option from "effect/Option";
2022
import * as Queue from "effect/Queue";
2123
import * as Ref from "effect/Ref";
2224
import * as Schema from "effect/Schema";
2325
import * as Stream from "effect/Stream";
2426
import * as SubscriptionRef from "effect/SubscriptionRef";
27+
import { AsyncResult, Atom, AtomRegistry } from "effect/reactivity";
2528
import * as TestClock from "effect/testing/TestClock";
2629
import { RpcClientError } from "effect/rpc";
2730
import * as Socket from "effect/socket/Socket";
@@ -32,11 +35,15 @@ import {
3235
type PreparedConnection,
3336
} from "../connection/model.ts";
3437
import * as EnvironmentSupervisor from "../connection/supervisor.ts";
38+
import * as EnvironmentRegistry from "../connection/registry.ts";
39+
import { RpcPermissionGuard } from "../rpc/client.ts";
3540
import * as Persistence from "../platform/persistence.ts";
3641
import type { WsRpcProtocolClient } from "../rpc/protocol.ts";
3742
import type { RpcSession } from "../rpc/session.ts";
3843
import {
3944
applyServerWelcomeEvent,
45+
createServerEnvironmentAtoms,
46+
serverUpdateStateAtom,
4047
makeEnvironmentServerWelcomeState,
4148
makeEnvironmentServerConfigState,
4249
isLegacyUpdateHandoffLoss,
@@ -216,6 +223,162 @@ describe("update restart reconnect nudges", () => {
216223
);
217224
});
218225

226+
describe("accepted server update recovery", () => {
227+
it.effect.each([
228+
{ loss: "interrupt", resumes: true },
229+
{ loss: "socket", resumes: true },
230+
{ loss: "server_error", resumes: false },
231+
{ loss: "no_progress", resumes: false },
232+
{ loss: "cancel", resumes: false },
233+
{ loss: "socket", method: "respawn", resumes: true },
234+
{ loss: "socket", method: "desktop-managed", resumes: false },
235+
] as const)("handles $loss with $method and recovery=$resumes", (scenario) =>
236+
Effect.gen(function* () {
237+
const { loss, resumes } = scenario;
238+
const method = "method" in scenario ? scenario.method : "boot-service";
239+
const observing = yield* Deferred.make<void>();
240+
const cancelFiber = yield* Deferred.make<Fiber.Fiber<unknown, unknown>>();
241+
const oldReadyObserved = yield* Deferred.make<void>();
242+
const readyEvents =
243+
yield* Queue.unbounded<Parameters<typeof matchesServerUpdateReadyEvent>[1]>();
244+
const config = {
245+
...CONFIG,
246+
environment: {
247+
...CONFIG.environment,
248+
serverVersion: "0.0.30",
249+
capabilities: { serverSelfUpdate: method, serverSelfUpdateProgress: true },
250+
},
251+
} as ServerConfig;
252+
const client = {
253+
[WS_METHODS.subscribeServerConfig]: () => Stream.never,
254+
[WS_METHODS.subscribeServerLifecycle]: () =>
255+
Stream.fromEffect(Deferred.succeed(observing, undefined)).pipe(
256+
Stream.drain,
257+
Stream.concat(Stream.fromQueue(readyEvents)),
258+
Stream.tap((event) =>
259+
event.payload.environment.serverVersion === "0.0.30"
260+
? Deferred.succeed(oldReadyObserved, undefined)
261+
: Effect.void,
262+
),
263+
),
264+
[WS_METHODS.serverUpdateServerWithProgress]: () =>
265+
(loss === "no_progress"
266+
? Stream.empty
267+
: Stream.make({ type: "progress" as const, stage: "downloading" as const })
268+
).pipe(
269+
Stream.concat(
270+
Stream.fromEffect(
271+
loss === "cancel"
272+
? Effect.fiber.pipe(
273+
Effect.flatMap((fiber) => Deferred.succeed(cancelFiber, fiber)),
274+
Effect.andThen(Effect.never),
275+
)
276+
: loss === "interrupt"
277+
? Effect.interrupt
278+
: Effect.fail(
279+
loss === "server_error"
280+
? new ServerSelfUpdateError({
281+
reason: "Archive verification failed.",
282+
})
283+
: new RpcClientError.RpcClientError({
284+
reason: new Socket.SocketCloseError({ code: 1006 }),
285+
}),
286+
),
287+
),
288+
),
289+
),
290+
} as unknown as WsRpcProtocolClient;
291+
const supervisor = EnvironmentSupervisor.EnvironmentSupervisor.of({
292+
target: TARGET,
293+
state: yield* SubscriptionRef.make(AVAILABLE_CONNECTION_STATE),
294+
session: yield* SubscriptionRef.make(Option.some(session(client))),
295+
prepared: yield* SubscriptionRef.make(Option.none<PreparedConnection>()),
296+
connect: Effect.void,
297+
disconnect: Effect.void,
298+
retryNow: Effect.void,
299+
});
300+
const run: EnvironmentRegistry.EnvironmentRegistry["Service"]["run"] = (_id, effect) =>
301+
Effect.provideService(effect, EnvironmentSupervisor.EnvironmentSupervisor, supervisor);
302+
const runStream: EnvironmentRegistry.EnvironmentRegistry["Service"]["runStream"] = (
303+
_id,
304+
stream,
305+
) => Stream.provideService(stream, EnvironmentSupervisor.EnvironmentSupervisor, supervisor);
306+
const environmentRegistry = EnvironmentRegistry.EnvironmentRegistry.of({
307+
run,
308+
runStream,
309+
followStream: runStream,
310+
stateChanges: () => Stream.never,
311+
retryNow: () => Effect.void,
312+
} as unknown as EnvironmentRegistry.EnvironmentRegistry["Service"]);
313+
const runtime = Atom.runtime(
314+
Layer.mergeAll(
315+
Layer.succeed(EnvironmentRegistry.EnvironmentRegistry, environmentRegistry),
316+
Layer.succeed(RpcPermissionGuard, { authorize: () => Effect.void }),
317+
Layer.succeed(Persistence.EnvironmentCacheStore, {
318+
loadServerConfig: () => Effect.succeedNone,
319+
} as unknown as Persistence.EnvironmentCacheStore["Service"]),
320+
),
321+
);
322+
const atoms = createServerEnvironmentAtoms(runtime, {
323+
initialConfigValueAtom: () => Atom.make(config),
324+
});
325+
const registry = yield* Effect.acquireRelease(Effect.sync(AtomRegistry.make), (value) =>
326+
Effect.sync(() => value.dispose()),
327+
);
328+
const pending = atoms.updateServer.run(registry, {
329+
environmentId: TARGET.environmentId,
330+
input: { targetVersion: "0.0.31" },
331+
});
332+
if (loss === "cancel") {
333+
yield* Fiber.interrupt(yield* Deferred.await(cancelFiber));
334+
const cancelled = yield* Effect.promise(() => pending);
335+
expect(cancelled._tag).toBe("Failure");
336+
if (cancelled._tag === "Failure")
337+
expect(Cause.hasInterruptsOnly(cancelled.cause)).toBe(true);
338+
expect(registry.get(serverUpdateStateAtom(TARGET.environmentId))).toEqual({
339+
status: "idle",
340+
});
341+
expect(yield* Deferred.isDone(observing)).toBe(false);
342+
return;
343+
}
344+
const first = yield* Effect.raceFirst(
345+
Effect.promise(() => pending).pipe(Effect.as("settled")),
346+
Deferred.await(observing).pipe(Effect.as("observing")),
347+
);
348+
if (!resumes) {
349+
expect(first).toBe("settled");
350+
expect(AsyncResult.isFailure(yield* Effect.promise(() => pending))).toBe(true);
351+
expect(registry.get(serverUpdateStateAtom(TARGET.environmentId))).toMatchObject({
352+
status: "failed",
353+
});
354+
return;
355+
}
356+
expect(first).toBe("observing");
357+
expect(registry.get(serverUpdateStateAtom(TARGET.environmentId))).toMatchObject({
358+
status: "running",
359+
stage: "resuming",
360+
});
361+
const ready = (serverVersion: string) =>
362+
({
363+
version: 1,
364+
sequence: 1,
365+
type: "ready",
366+
payload: { at: "2026-09-01T00:00:00.000Z", environment: { serverVersion } },
367+
}) as Parameters<typeof matchesServerUpdateReadyEvent>[1];
368+
yield* Queue.offer(readyEvents, ready("0.0.30"));
369+
yield* Deferred.await(oldReadyObserved);
370+
expect(registry.get(serverUpdateStateAtom(TARGET.environmentId))).toMatchObject({
371+
status: "running",
372+
});
373+
yield* Queue.offer(readyEvents, ready("0.0.31"));
374+
expect(AsyncResult.isSuccess(yield* Effect.promise(() => pending))).toBe(true);
375+
expect(registry.get(serverUpdateStateAtom(TARGET.environmentId))).toEqual({
376+
status: "idle",
377+
});
378+
}).pipe(Effect.scoped),
379+
);
380+
});
381+
219382
const makeConfigPersistenceHarness = Effect.fn("test.makeConfigPersistenceHarness")(function* (
220383
saveServerConfig: Persistence.EnvironmentCacheStore["Service"]["saveServerConfig"],
221384
) {

0 commit comments

Comments
 (0)