Skip to content

Commit 808d685

Browse files
t3dotggclaude
andcommitted
fix(server): keep the managed tunnel across remote update restarts
Measured on a live update: the server was back in ~9s but stayed unreachable for ~96s, because releasing the tunnel on shutdown forces the replacement tunnel's hostname route through 1-2 minutes of edge propagation. An update handoff always brings a server right back (new version or rollback), so the tunnel is never orphaned; skip the release when the launcher state file records a pending update. The next boot respawns the connector from the stored config against the same tunnel and is reachable as soon as it connects. Co-Authored-By: Claude Fable 5 <[email protected]>
1 parent 80720ad commit 808d685

2 files changed

Lines changed: 102 additions & 0 deletions

File tree

‎apps/server/src/cloud/http.test.ts‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
import * as NodeServices from "@effect/platform-node/NodeServices";
22
import { describe, expect, it } from "@effect/vitest";
33
import * as Effect from "effect/Effect";
4+
import * as FileSystem from "effect/FileSystem";
45
import * as Option from "effect/Option";
6+
import * as Path from "effect/Path";
57
import * as PlatformError from "effect/PlatformError";
68
import * as Tracer from "effect/Tracer";
79
import {
@@ -15,6 +17,7 @@ import { EnvironmentId } from "@t3tools/contracts";
1517
import { RelayClientTracer } from "@t3tools/shared/relayTracing";
1618
import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts";
1719
import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
20+
import * as ServerConfigModule from "../config.ts";
1821
import * as ServerEnvironment from "../environment/ServerEnvironment.ts";
1922
import { CLOUD_CLI_DESIRED_LINK_SECRET } from "./CliState.ts";
2023
import * as CliTokenManager from "./CliTokenManager.ts";
@@ -256,6 +259,21 @@ describe("releaseManagedTunnelOnShutdown", () => {
256259
readonly respond?: () => Response;
257260
}
258261

262+
// Writes the launcher's durable state file into this test's baseDir; the
263+
// release reads it to detect an in-flight update handoff.
264+
const writeServiceState = (update: Record<string, unknown>) =>
265+
Effect.gen(function* () {
266+
const fs = yield* FileSystem.FileSystem;
267+
const path = yield* Path.Path;
268+
const config = yield* ServerConfigModule.ServerConfig;
269+
const statePath = path.join(config.baseDir, "runtime", "service-state.json");
270+
yield* fs.makeDirectory(path.dirname(statePath), { recursive: true });
271+
yield* fs.writeFileString(
272+
statePath,
273+
JSON.stringify({ version: 1, activeVersion: "0.0.30", update }),
274+
);
275+
});
276+
259277
const provideReleaseHarness =
260278
(harness: ReleaseHarness) =>
261279
<A, E, R>(effect: Effect.Effect<A, E, R>) =>
@@ -306,6 +324,11 @@ describe("releaseManagedTunnelOnShutdown", () => {
306324
}),
307325
),
308326
),
327+
// The release consults the launcher state file under the configured
328+
// baseDir, so every harness run gets a scoped temp baseDir.
329+
Effect.provide(ServerConfigModule.layerTest("/", { prefix: "t3-http-release-test-" })),
330+
Effect.provide(NodeServices.layer),
331+
Effect.scoped,
309332
);
310333

311334
// The persisted state of a CLI-managed link whose tunnel is releasable.
@@ -390,6 +413,54 @@ describe("releaseManagedTunnelOnShutdown", () => {
390413
}).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
391414
});
392415

416+
it.effect("keeps the tunnel when shutdown hands off to a pending update", () => {
417+
const { store, values } = makeMemorySecretStore(managedLinkSecrets);
418+
const applyConfigCalls: Array<unknown> = [];
419+
const requests: Array<HttpClientRequest.HttpClientRequest> = [];
420+
421+
return Effect.gen(function* () {
422+
yield* writeServiceState({
423+
id: "update-1",
424+
fromVersion: "0.0.30",
425+
targetVersion: "0.0.31",
426+
dbPath: "/tmp/state.sqlite",
427+
status: "pending",
428+
});
429+
430+
const released = yield* releaseManagedTunnelOnShutdown();
431+
432+
// The launcher restarts a server immediately, so the tunnel is not
433+
// orphaned; keeping it avoids the hostname route re-propagation that
434+
// dominates update downtime. The stored config must survive so the
435+
// next boot respawns the connector against the same tunnel.
436+
expect(released).toBe(false);
437+
expect(applyConfigCalls).toEqual([]);
438+
expect(requests).toEqual([]);
439+
expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(true);
440+
}).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
441+
});
442+
443+
it.effect("still releases when the recorded update already settled", () => {
444+
const { store, values } = makeMemorySecretStore(managedLinkSecrets);
445+
const applyConfigCalls: Array<unknown> = [];
446+
const requests: Array<HttpClientRequest.HttpClientRequest> = [];
447+
448+
return Effect.gen(function* () {
449+
yield* writeServiceState({
450+
id: "update-1",
451+
fromVersion: "0.0.30",
452+
targetVersion: "0.0.31",
453+
status: "committed",
454+
});
455+
456+
const released = yield* releaseManagedTunnelOnShutdown();
457+
458+
expect(released).toBe(true);
459+
expect(requests).toHaveLength(1);
460+
expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(false);
461+
}).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
462+
});
463+
393464
it.effect("keeps a runtime config that a fast restart replaced mid-release", () => {
394465
const { store, values } = makeMemorySecretStore(managedLinkSecrets);
395466
const applyConfigCalls: Array<unknown> = [];

‎apps/server/src/cloud/http.ts‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,9 @@ import * as DateTime from "effect/DateTime";
4646
import * as Crypto from "effect/Crypto";
4747
import * as Duration from "effect/Duration";
4848
import * as Effect from "effect/Effect";
49+
import * as FileSystem from "effect/FileSystem";
4950
import * as Option from "effect/Option";
51+
import * as Path from "effect/Path";
5052
import * as Schema from "effect/Schema";
5153
import * as HttpEffect from "effect/unstable/http/HttpEffect";
5254
import { HttpServerRequest, HttpServerResponse } from "effect/unstable/http";
@@ -56,8 +58,10 @@ import * as HttpApiBuilder from "effect/unstable/httpapi/HttpApiBuilder";
5658
import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts";
5759
import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
5860
import { requireEnvironmentScope } from "../auth/http.ts";
61+
import * as ServerConfig from "../config.ts";
5962
import * as ServerEnvironment from "../environment/ServerEnvironment.ts";
6063
import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts";
64+
import { SERVICE_STATE_FILE, serviceStateHasPendingUpdate } from "./serviceProtocol.ts";
6165
import {
6266
CLOUD_ENDPOINT_RUNTIME_CONFIG,
6367
CLOUD_LINKED_USER_ID,
@@ -627,6 +631,20 @@ export const reconcileDesiredCloudLink = Effect.fn("environment.cloud.reconcileD
627631
},
628632
);
629633

634+
// The launcher records an in-flight update in its durable state file. Read
635+
// straight from disk: the launcher owns that file, and this runs while the
636+
// server is tearing down. Unreadable or unparseable state means no pending
637+
// update — the release then proceeds as before.
638+
const pendingServiceUpdateExists = Effect.gen(function* () {
639+
const config = yield* ServerConfig.ServerConfig;
640+
const fs = yield* FileSystem.FileSystem;
641+
const path = yield* Path.Path;
642+
const stateText = yield* fs
643+
.readFileString(path.join(config.baseDir, "runtime", SERVICE_STATE_FILE))
644+
.pipe(Effect.option);
645+
return Option.isSome(stateText) && serviceStateHasPendingUpdate(stateText.value);
646+
});
647+
630648
// Cloudflare bills per provisioned tunnel, so an environment that goes offline
631649
// must not leave its tunnel behind. Releasing deletes only the tunnel — the
632650
// relay keeps the link and its hostname reservation, and the next startup's
@@ -649,6 +667,19 @@ export const releaseManagedTunnelOnShutdown = Effect.fn(
649667
if (!(yield* readCliDesiredCloudLink) || (yield* readCliDesiredLinkMode) !== "managed") {
650668
return false;
651669
}
670+
// A shutdown that hands off to a pending remote update is not the
671+
// environment going offline: the launcher immediately brings a server back
672+
// (the new version, or the old one after a rollback). Deleting the tunnel
673+
// here forces that server to provision a replacement UUID, and the public
674+
// hostname's route to the new tunnel takes 1-2 minutes to propagate — the
675+
// dominant cost of an update restart. Keep the tunnel instead: the next
676+
// boot respawns the connector from the stored config and is reachable as
677+
// soon as it connects, and the reconcile confirms the still-live tunnel
678+
// without replacing it.
679+
if (yield* pendingServiceUpdateExists) {
680+
yield* Effect.logInfo("Keeping the managed tunnel across the update restart");
681+
return false;
682+
}
652683
const token = yield* dependencies.cliTokenManager.getExisting;
653684
if (Option.isNone(token)) {
654685
return false;

0 commit comments

Comments
 (0)