Skip to content

Commit 23d98c2

Browse files
committed
fix(server): keep tunnels during remote desktop updates
1 parent 935c55b commit 23d98c2

6 files changed

Lines changed: 218 additions & 41 deletions

File tree

‎apps/server/src/cloud/http.test.ts‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -454,6 +454,21 @@ describe("releaseManagedTunnelOnShutdown", () => {
454454
}).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
455455
});
456456

457+
it.effect("keeps the tunnel while a remote desktop update is restarting", () => {
458+
const { store, values } = makeMemorySecretStore(managedLinkSecrets);
459+
const applyConfigCalls: Array<unknown> = [];
460+
const requests: Array<HttpClientRequest.HttpClientRequest> = [];
461+
462+
return Effect.gen(function* () {
463+
const released = yield* releaseManagedTunnelOnShutdown(Effect.succeed(true));
464+
465+
expect(released).toBe(false);
466+
expect(applyConfigCalls).toEqual([]);
467+
expect(requests).toEqual([]);
468+
expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(true);
469+
}).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
470+
});
471+
457472
it.effect("still releases a pending update when the launcher is stopping", () => {
458473
// `t3 service uninstall` or `systemctl stop` during the pending window:
459474
// the launcher writes its stop marker before signalling the child, so no

‎apps/server/src/cloud/http.ts‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -669,7 +669,7 @@ const pendingUpdateHandoffExists = Effect.gen(function* () {
669669
// link reconcile provisions a replacement tunnel under the same URL.
670670
export const releaseManagedTunnelOnShutdown = Effect.fn(
671671
"environment.cloud.releaseManagedTunnelOnShutdown",
672-
)(function* () {
672+
)(function* (desktopUpdatePending: Effect.Effect<boolean> = Effect.succeed(false)) {
673673
const dependencies = yield* cloudHttpDependencies;
674674
// Only a managed link stores a runtime config; publish-only links have no
675675
// tunnel to release.
@@ -686,15 +686,15 @@ export const releaseManagedTunnelOnShutdown = Effect.fn(
686686
return false;
687687
}
688688
// A shutdown that hands off to a pending remote update is not the
689-
// environment going offline: the launcher immediately brings a server back
690-
// (the new version, or the old one after a rollback). Deleting the tunnel
691-
// here forces that server to provision a replacement UUID, and the public
692-
// hostname's route to the new tunnel takes 1-2 minutes to propagate — the
693-
// dominant cost of an update restart. Keep the tunnel instead: the next
689+
// environment going offline: the service launcher or desktop app brings a
690+
// server back. Desktop installs do not write the launcher's state file.
691+
// Deleting the tunnel forces that server to provision a replacement UUID.
692+
// The hostname's route to the new tunnel takes 1-2 minutes to propagate.
693+
// Keep the tunnel instead: the next
694694
// boot respawns the connector from the stored config and is reachable as
695695
// soon as it connects, and the reconcile confirms the still-live tunnel
696696
// without replacing it.
697-
if (yield* pendingUpdateHandoffExists) {
697+
if ((yield* desktopUpdatePending) || (yield* pendingUpdateHandoffExists)) {
698698
yield* Effect.logInfo("Keeping the managed tunnel across the update restart");
699699
return false;
700700
}

‎apps/server/src/cloud/selfUpdate.test.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,9 @@ const makeHarness = Effect.fn("test.make_self_update_harness")(function* (
117117
DesktopAppUpdate.DesktopAppUpdate,
118118
options.desktopAppUpdate ?? {
119119
available: false,
120+
isRestartPending: Effect.succeed(false),
120121
run: () => Effect.die("unexpected desktop app update run"),
122+
commit: () => Effect.die("unexpected desktop app update commit"),
121123
},
122124
),
123125
Effect.provideService(HttpClient.HttpClient, releaseHttpClient(order)),
@@ -377,6 +379,7 @@ it.layer(NodeServices.layer)("server self update", (it) => {
377379
mode: "desktop",
378380
desktopAppUpdate: {
379381
available: true,
382+
isRestartPending: Effect.succeed(false),
380383
run: (reportProgress) =>
381384
reportProgress("downloading").pipe(
382385
Effect.andThen(reportProgress("installing")),

‎apps/server/src/desktopUpdate/DesktopAppUpdate.test.ts‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import * as Fiber from "effect/Fiber";
88
import * as Layer from "effect/Layer";
99
import * as Option from "effect/Option";
1010
import * as Stream from "effect/Stream";
11+
import * as TestClock from "effect/testing/TestClock";
1112

1213
import * as ServerConfig from "../config.ts";
1314
import * as DesktopTelemetryReceiver from "../resourceTelemetry/DesktopTelemetryReceiver.ts";
@@ -60,6 +61,7 @@ interface HarnessOptions {
6061
The stream ends after the last one unless `keepOpen` is set. */
6162
readonly reports?: (requestId: string) => readonly DesktopUpdateStatusReport[];
6263
readonly keepOpen?: boolean;
64+
readonly receiver?: Partial<DesktopTelemetryReceiver.DesktopTelemetryReceiver["Service"]>;
6365
}
6466

6567
const makeHarness = Effect.fn("test.make_desktop_app_update_harness")(function* (
@@ -98,6 +100,7 @@ const makeHarness = Effect.fn("test.make_desktop_app_update_harness")(function*
98100
latest: Option.none<DesktopUpdateStatusReport>(),
99101
changes,
100102
}),
103+
...options.receiver,
101104
}),
102105
ServerConfig.layer(config),
103106
),
@@ -144,6 +147,7 @@ it.layer(NodeServices.layer)("desktop app update", (it) => {
144147
});
145148
// "downloading" is not repeated for every download report.
146149
expect(stages).toEqual(["downloading", "installing"]);
150+
expect(yield* service.isRestartPending).toBe(false);
147151

148152
// Success releases the in-flight guard: if the desktop rejected the
149153
// install after reporting, the server must accept a retry instead of
@@ -203,6 +207,111 @@ it.layer(NodeServices.layer)("desktop app update", (it) => {
203207
expect(
204208
(yield* service.commit(prepared.desktopUpdateToken ?? "missing").pipe(Effect.flip)).reason,
205209
).toBe("installer refused");
210+
expect(yield* service.isRestartPending).toBe(false);
211+
}),
212+
);
213+
214+
it.effect("retains the tunnel handoff after transport loss, then expires it", () =>
215+
Effect.gen(function* () {
216+
const accepted = yield* Deferred.make<void>();
217+
const { service } = yield* makeHarness({ keepOpen: true });
218+
const commit = yield* service
219+
.commit("update-1", () => Deferred.succeed(accepted, undefined).pipe(Effect.asVoid))
220+
.pipe(Effect.forkChild);
221+
yield* Deferred.await(accepted);
222+
expect(yield* service.isRestartPending).toBe(true);
223+
224+
yield* Fiber.interrupt(commit);
225+
expect(yield* service.isRestartPending).toBe(true);
226+
227+
yield* TestClock.adjust("2 minutes");
228+
expect(yield* service.isRestartPending).toBe(false);
229+
}),
230+
);
231+
232+
it.effect("clears a failed install even after its remote caller disconnects", () =>
233+
Effect.gen(function* () {
234+
const accepted = yield* Deferred.make<void>();
235+
let latest = Option.none<DesktopUpdateStatusReport>();
236+
const { service } = yield* makeHarness({
237+
receiver: {
238+
desktopUpdates: Effect.sync(() => ({ latest, changes: Stream.never })),
239+
},
240+
});
241+
const commit = yield* service
242+
.commit("update-1", () => Deferred.succeed(accepted, undefined).pipe(Effect.asVoid))
243+
.pipe(Effect.forkChild);
244+
yield* Deferred.await(accepted);
245+
yield* Fiber.interrupt(commit);
246+
latest = Option.some(report("update-1", makeState(), { outcome: "failed" }));
247+
248+
expect(yield* service.isRestartPending).toBe(false);
249+
}),
250+
);
251+
252+
it.effect("clears the handoff when the desktop control write fails", () =>
253+
Effect.gen(function* () {
254+
const { service } = yield* makeHarness({
255+
receiver: {
256+
commitDesktopUpdate: () =>
257+
Effect.fail(
258+
new DesktopTelemetryReceiver.DesktopTelemetryControlStalled({
259+
fd: 5,
260+
remainingBytes: 1,
261+
}),
262+
),
263+
},
264+
});
265+
266+
yield* service.commit("update-1").pipe(Effect.flip);
267+
expect(yield* service.isRestartPending).toBe(false);
268+
}),
269+
);
270+
271+
it.effect("clears the handoff when installation times out", () =>
272+
Effect.gen(function* () {
273+
const waiting = yield* Deferred.make<void>();
274+
const { service } = yield* makeHarness({
275+
receiver: {
276+
desktopUpdates: Effect.succeed({
277+
latest: Option.none(),
278+
changes: Stream.fromEffect(Deferred.succeed(waiting, undefined)).pipe(
279+
Stream.drain,
280+
Stream.concat(Stream.never),
281+
),
282+
}),
283+
},
284+
});
285+
const commit = yield* service.commit("update-1").pipe(Effect.flip, Effect.forkChild);
286+
yield* Deferred.await(waiting);
287+
yield* TestClock.adjust("2 minutes");
288+
289+
expect((yield* Fiber.join(commit)).reason).toBe(
290+
"The desktop app did not report an install result in time.",
291+
);
292+
expect(yield* service.isRestartPending).toBe(false);
293+
}),
294+
);
295+
296+
it.effect("does not let a rejected commit clear another install's handoff", () =>
297+
Effect.gen(function* () {
298+
const accepted = yield* Deferred.make<void>();
299+
const { service } = yield* makeHarness({
300+
receiver: {
301+
desktopUpdates: Effect.succeed({
302+
latest: Option.some(report("rejected", makeState(), { outcome: "failed" })),
303+
changes: Stream.never,
304+
}),
305+
},
306+
});
307+
const commit = yield* service
308+
.commit("update-1", () => Deferred.succeed(accepted, undefined).pipe(Effect.asVoid))
309+
.pipe(Effect.forkChild);
310+
yield* Deferred.await(accepted);
311+
yield* Fiber.interrupt(commit);
312+
yield* service.commit("rejected").pipe(Effect.flip);
313+
314+
expect(yield* service.isRestartPending).toBe(true);
206315
}),
207316
);
208317

‎apps/server/src/desktopUpdate/DesktopAppUpdate.ts‎

Lines changed: 79 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,13 @@ import {
55
type ServerSelfUpdateProgressStage,
66
type ServerSelfUpdateResult,
77
} from "@t3tools/contracts";
8+
import * as Cause from "effect/Cause";
9+
import * as Clock from "effect/Clock";
810
import * as Context from "effect/Context";
911
import * as Crypto from "effect/Crypto";
1012
import * as Duration from "effect/Duration";
1113
import * as Effect from "effect/Effect";
14+
import * as HashMap from "effect/HashMap";
1215
import * as Layer from "effect/Layer";
1316
import * as Option from "effect/Option";
1417
import * as Ref from "effect/Ref";
@@ -46,6 +49,8 @@ export class DesktopAppUpdate extends Context.Service<
4649
/** True when this server was spawned by a desktop app that can be
4750
driven over the telemetry control channel. */
4851
readonly available: boolean;
52+
/** Keeps the managed tunnel while an accepted remote install restarts this server. */
53+
readonly isRestartPending: Effect.Effect<boolean>;
4954
/** Checks and downloads through the desktop app, then returns a token
5055
while this server is still connected. `commit` starts installation. */
5156
readonly run: (
@@ -67,6 +72,27 @@ export const make = Effect.fn("desktopUpdate.desktopAppUpdate.make")(function* (
6772
const crypto = yield* Crypto.Crypto;
6873
const receiver = yield* DesktopTelemetryReceiver.DesktopTelemetryReceiver;
6974
const inFlight = yield* Ref.make(false);
75+
// The RPC can disappear before shutdown cleanup runs. Retain accepted
76+
// handoffs across that interruption, but bound them by the install deadline.
77+
const pendingRestarts = yield* Ref.make(HashMap.empty<string, number>());
78+
79+
const isRestartPending = Effect.scoped(
80+
Effect.gen(function* () {
81+
const now = yield* Clock.currentTimeMillis;
82+
const { latest } = yield* receiver.desktopUpdates;
83+
const failedRequestId =
84+
Option.isSome(latest) && latest.value.outcome === "failed"
85+
? latest.value.requestId
86+
: undefined;
87+
const pending = yield* Ref.updateAndGet(pendingRestarts, (requests) =>
88+
HashMap.filter(
89+
requests,
90+
(deadline, requestId) => deadline > now && requestId !== failedRequestId,
91+
),
92+
);
93+
return HashMap.size(pending) > 0;
94+
}),
95+
);
7096

7197
const available = config.mode === "desktop" && config.desktopTelemetryControlFd !== undefined;
7298
const failWith = (reason: string, cause?: unknown) =>
@@ -191,41 +217,62 @@ export const make = Effect.fn("desktopUpdate.desktopAppUpdate.make")(function* (
191217
if (!available) {
192218
return yield* failWith("This server cannot commit a desktop app update.");
193219
}
194-
const terminal = yield* Effect.scoped(
195-
Effect.gen(function* () {
196-
const { latest, changes } = yield* receiver.desktopUpdates;
197-
const reports = Option.match(latest, {
198-
onNone: () => changes,
199-
onSome: (report) => Stream.concat(Stream.make(report), changes),
200-
});
201-
yield* Effect.uninterruptible(
202-
receiver.commitDesktopUpdate(requestId).pipe(
203-
Effect.mapError((error) => failWith("Could not reach the T3 Code desktop app.", error)),
204-
Effect.tap(() => onHandoffAccepted()),
205-
),
206-
);
207-
return yield* reports.pipe(
208-
Stream.filter((report) => report.requestId === requestId && report.outcome === "failed"),
209-
Stream.runHead,
210-
Effect.timeout(DESKTOP_INSTALL_TIMEOUT),
211-
Effect.catchTags({
212-
TimeoutError: () =>
213-
failWith("The desktop app did not report an install result in time."),
214-
}),
215-
);
216-
}),
217-
);
218-
if (Option.isNone(terminal)) {
219-
return yield* failWith("The desktop app stopped reporting the install.");
220-
}
221-
return yield* failWith(
222-
terminal.value.reason ??
223-
terminal.value.state.message ??
224-
"The desktop app failed to install the update.",
220+
let handoffAccepted = false;
221+
return yield* Effect.gen(function* () {
222+
const terminal = yield* Effect.scoped(
223+
Effect.gen(function* () {
224+
const { latest, changes } = yield* receiver.desktopUpdates;
225+
const reports = Option.match(latest, {
226+
onNone: () => changes,
227+
onSome: (report) => Stream.concat(Stream.make(report), changes),
228+
});
229+
yield* Effect.uninterruptible(
230+
Effect.gen(function* () {
231+
const now = yield* Clock.currentTimeMillis;
232+
yield* Ref.update(
233+
pendingRestarts,
234+
HashMap.set(requestId, now + Duration.toMillis(DESKTOP_INSTALL_TIMEOUT)),
235+
);
236+
yield* receiver.commitDesktopUpdate(requestId);
237+
handoffAccepted = true;
238+
}).pipe(
239+
Effect.mapError((error) =>
240+
failWith("Could not reach the T3 Code desktop app.", error),
241+
),
242+
Effect.tap(() => onHandoffAccepted()),
243+
),
244+
);
245+
return yield* reports.pipe(
246+
Stream.filter(
247+
(report) => report.requestId === requestId && report.outcome === "failed",
248+
),
249+
Stream.runHead,
250+
Effect.timeout(DESKTOP_INSTALL_TIMEOUT),
251+
Effect.catchTags({
252+
TimeoutError: () =>
253+
failWith("The desktop app did not report an install result in time."),
254+
}),
255+
);
256+
}),
257+
);
258+
if (Option.isNone(terminal)) {
259+
return yield* failWith("The desktop app stopped reporting the install.");
260+
}
261+
return yield* failWith(
262+
terminal.value.reason ??
263+
terminal.value.state.message ??
264+
"The desktop app failed to install the update.",
265+
);
266+
}).pipe(
267+
Effect.onExit((exit) =>
268+
exit._tag === "Failure" && handoffAccepted && Cause.hasInterruptsOnly(exit.cause)
269+
? Effect.void
270+
: Ref.update(pendingRestarts, HashMap.remove(requestId)),
271+
),
225272
);
226273
});
227274

228-
return DesktopAppUpdate.of({ available, run, commit });
275+
return DesktopAppUpdate.of({ available, isRestartPending, run, commit });
229276
});
230277

231278
export const layer = Layer.effect(DesktopAppUpdate, make());

‎apps/server/src/server.ts‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -705,7 +705,10 @@ const makeServerLayer = Layer.unwrap(
705705
yield* Deferred.succeed(cloudLinkParked, undefined).pipe(Effect.orDie);
706706
return;
707707
}
708-
const releaseManagedTunnel = releaseManagedTunnelOnShutdown().pipe(
708+
const desktopUpdate = yield* DesktopAppUpdate.DesktopAppUpdate;
709+
const releaseManagedTunnel = releaseManagedTunnelOnShutdown(
710+
desktopUpdate.isRestartPending,
711+
).pipe(
709712
Effect.timeout("10 seconds"),
710713
Effect.tap((released) =>
711714
released ? Effect.logInfo("Released the managed tunnel on shutdown") : Effect.void,
@@ -789,7 +792,7 @@ const makeServerLayer = Layer.unwrap(
789792
httpListeningLayer,
790793
runtimeStateLayer.pipe(Layer.provide(launcherLayer)),
791794
tailscaleServeLayer,
792-
cloudDesiredLinkReconcileLayer,
795+
cloudDesiredLinkReconcileLayer.pipe(Layer.provide(DesktopAppUpdateLayerLive)),
793796
);
794797

795798
return serverApplicationLayer.pipe(

0 commit comments

Comments
 (0)