Skip to content

Commit 6a0992d

Browse files
committed
fix(relay): harden managed tunnel recovery
1 parent 8c6e414 commit 6a0992d

33 files changed

Lines changed: 1900 additions & 299 deletions

‎.github/workflows/deploy-relay.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ jobs:
2828
RELAY_DOMAIN: ${{ vars.RELAY_DOMAIN }}
2929
RELAY_API_ZONE_NAME: ${{ vars.RELAY_API_ZONE_NAME }}
3030
RELAY_TUNNEL_ZONE_NAME: ${{ vars.RELAY_TUNNEL_ZONE_NAME }}
31+
RELAY_TUNNEL_CLEANUP_MODE: ${{ vars.RELAY_TUNNEL_CLEANUP_MODE }}
3132
CLERK_PUBLISHABLE_KEY: ${{ vars.CLERK_PUBLISHABLE_KEY }}
3233
CLERK_JWT_AUDIENCE: ${{ vars.CLERK_JWT_AUDIENCE }}
3334
APNS_ENVIRONMENT: ${{ vars.APNS_ENVIRONMENT }}

‎apps/server/src/cloud/CliState.test.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
88
import { ServerConfig } from "../config.ts";
99
import * as CliState from "./CliState.ts";
1010
import {
11+
CLOUD_ENDPOINT_CONFIRMED_ORIGIN,
1112
CLOUD_ENDPOINT_RUNTIME_CONFIG,
1213
CLOUD_LINKED_USER_ID,
1314
CLOUD_MINT_PUBLIC_KEY,
@@ -24,6 +25,7 @@ const persistedCloudLinkSecrets = [
2425
RELAY_ENVIRONMENT_CREDENTIAL_SECRET,
2526
CLOUD_MINT_PUBLIC_KEY,
2627
CLOUD_ENDPOINT_RUNTIME_CONFIG,
28+
CLOUD_ENDPOINT_CONFIRMED_ORIGIN,
2729
PUBLISH_AGENT_ACTIVITY_SECRET,
2830
] as const;
2931

‎apps/server/src/cloud/CliState.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import * as Option from "effect/Option";
33

44
import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
55
import {
6+
CLOUD_ENDPOINT_CONFIRMED_ORIGIN,
67
CLOUD_ENDPOINT_RUNTIME_CONFIG,
78
CLOUD_LINKED_USER_ID,
89
CLOUD_MINT_PUBLIC_KEY,
@@ -67,6 +68,7 @@ export const clearPersistedCloudLink = Effect.gen(function* () {
6768
secrets.remove(RELAY_ENVIRONMENT_CREDENTIAL_SECRET),
6869
secrets.remove(CLOUD_MINT_PUBLIC_KEY),
6970
secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG),
71+
secrets.remove(CLOUD_ENDPOINT_CONFIRMED_ORIGIN),
7072
secrets.remove(PUBLISH_AGENT_ACTIVITY_SECRET),
7173
],
7274
{ concurrency: "unbounded" },

‎apps/server/src/cloud/ManagedEndpointRuntime.test.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -333,8 +333,8 @@ describe("CloudManagedEndpointRuntime", () => {
333333

334334
expect(spawned.map((command) => command.command)).toEqual(["cloudflared", "cloudflared"]);
335335
expect(spawned.map((command) => command.args)).toEqual([
336-
["tunnel", "run"],
337-
["tunnel", "run"],
336+
["tunnel", "--no-autoupdate", "--loglevel", "info", "--output", "default", "run"],
337+
["tunnel", "--no-autoupdate", "--loglevel", "info", "--output", "default", "run"],
338338
]);
339339
expect(spawned.map((command) => command.options.env?.TUNNEL_TOKEN)).toEqual([
340340
"token-1",

‎apps/server/src/cloud/ManagedEndpointRuntime.ts‎

Lines changed: 13 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@ import * as Context from "effect/Context";
44
import * as Effect from "effect/Effect";
55
import * as Exit from "effect/Exit";
66
import * as Layer from "effect/Layer";
7-
import * as Option from "effect/Option";
87
import * as Queue from "effect/Queue";
98
import * as Ref from "effect/Ref";
109
import * as Result from "effect/Result";
@@ -14,22 +13,6 @@ import * as Stream from "effect/Stream";
1413
import * as ChildProcess from "effect/unstable/process/ChildProcess";
1514
import * as ChildProcessSpawner from "effect/unstable/process/ChildProcessSpawner";
1615

17-
import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
18-
import { CLOUD_ENDPOINT_RUNTIME_CONFIG, decodeRuntimeConfig } from "./config.ts";
19-
20-
function bytesToString(bytes: Uint8Array): string {
21-
return new TextDecoder().decode(bytes);
22-
}
23-
24-
const readRuntimeConfig = Effect.gen(function* () {
25-
const secrets = yield* ServerSecretStore.ServerSecretStore;
26-
const bytes = yield* secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG);
27-
if (Option.isNone(bytes)) {
28-
return null;
29-
}
30-
return Option.getOrNull(decodeRuntimeConfig(bytesToString(bytes.value)));
31-
});
32-
3316
export type CloudManagedEndpointRuntimeStatus =
3417
| {
3518
readonly status: "disabled";
@@ -275,16 +258,20 @@ export const make = Effect.gen(function* () {
275258
const connectorScope = yield* Scope.make("sequential");
276259
const child = yield* spawner
277260
.spawn(
278-
ChildProcess.make(executable.executablePath, ["tunnel", "run"], {
279-
detached: false,
280-
env: {
281-
...process.env,
282-
TUNNEL_TOKEN: config.connectorToken,
261+
ChildProcess.make(
262+
executable.executablePath,
263+
["tunnel", "--no-autoupdate", "--loglevel", "info", "--output", "default", "run"],
264+
{
265+
detached: false,
266+
env: {
267+
...process.env,
268+
TUNNEL_TOKEN: config.connectorToken,
269+
},
270+
shell: false,
271+
stderr: "pipe",
272+
stdout: "pipe",
283273
},
284-
shell: false,
285-
stderr: "pipe",
286-
stdout: "pipe",
287-
}),
274+
),
288275
)
289276
.pipe(
290277
Effect.provideService(Scope.Scope, connectorScope),
@@ -361,14 +348,6 @@ export const make = Effect.gen(function* () {
361348
withLinkStateLock: linkStateSemaphore.withPermits(1),
362349
});
363350

364-
const initialConfig = yield* readRuntimeConfig.pipe(
365-
Effect.catch((cause) =>
366-
Effect.logWarning("Failed to read managed endpoint runtime config", { cause }).pipe(
367-
Effect.as(null),
368-
),
369-
),
370-
);
371-
yield* runtime.applyConfig(initialConfig);
372351
yield* Effect.addFinalizer(() => runtime.applyConfig(null));
373352
return runtime;
374353
});

‎apps/server/src/cloud/config.ts‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
1-
import { RelayManagedEndpointRuntimeConfig } from "@t3tools/contracts/relay";
1+
import {
2+
RelayManagedEndpointOrigin,
3+
RelayManagedEndpointRuntimeConfig,
4+
} from "@t3tools/contracts/relay";
25
import * as Effect from "effect/Effect";
36
import * as Option from "effect/Option";
47
import * as Schema from "effect/Schema";
@@ -7,6 +10,7 @@ import type * as ServerSecretStore from "../auth/ServerSecretStore.ts";
710

811
export const CLOUD_MINT_PUBLIC_KEY = "cloud-mint-ed25519-public-key";
912
export const CLOUD_ENDPOINT_RUNTIME_CONFIG = "cloud-endpoint-runtime-config";
13+
export const CLOUD_ENDPOINT_CONFIRMED_ORIGIN = "cloud-endpoint-confirmed-origin";
1014
export const CLOUD_LINKED_USER_ID = "cloud-linked-user-id";
1115
export const RELAY_URL_SECRET = "cloud-relay-url";
1216
export const RELAY_ISSUER_SECRET = "cloud-relay-issuer";
@@ -21,6 +25,19 @@ export const decodeRuntimeConfig = Schema.decodeUnknownOption(
2125
Schema.fromJsonString(RelayManagedEndpointRuntimeConfig),
2226
);
2327

28+
export const ManagedEndpointConfirmedOrigin = Schema.Struct({
29+
config: RelayManagedEndpointRuntimeConfig,
30+
origin: RelayManagedEndpointOrigin,
31+
});
32+
33+
export const encodeConfirmedOriginJson = Schema.encodeEffect(
34+
Schema.fromJsonString(ManagedEndpointConfirmedOrigin),
35+
);
36+
37+
export const decodeConfirmedOrigin = Schema.decodeUnknownOption(
38+
Schema.fromJsonString(ManagedEndpointConfirmedOrigin),
39+
);
40+
2441
export function isAgentActivityPublishingEnabledValue(value: string | null): boolean {
2542
return value === "true";
2643
}

‎apps/server/src/cloud/http.test.ts‎

Lines changed: 134 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ import {
3636
type RelayLinkProofRequest,
3737
} from "@t3tools/contracts/relay";
3838
import {
39+
CLOUD_ENDPOINT_CONFIRMED_ORIGIN,
3940
CLOUD_ENDPOINT_RUNTIME_CONFIG,
4041
CLOUD_LINKED_USER_ID,
4142
decodeRuntimeConfig,
@@ -48,9 +49,11 @@ import {
4849
linkProofScopes,
4950
pendingServiceUpdateExists,
5051
reconcileDesiredCloudLink,
52+
reconcileDesiredCloudLinkIfStillDesired,
5153
recoverManagedCloudTunnel,
5254
registerManagedCloudTunnelRecovery,
5355
releaseManagedTunnelOnShutdown,
56+
startManagedCloudTunnelIfOriginConfirmed,
5457
} from "./http.ts";
5558
import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts";
5659
import { traceAuthenticatedRelayRequest, traceRelayRequest } from "./traceRelayRequest.ts";
@@ -379,10 +382,27 @@ describe("releaseManagedTunnelOnShutdown", () => {
379382
// The persisted state of a CLI-managed link whose tunnel is releasable.
380383
const managedLinkSecrets = [
381384
[CLOUD_ENDPOINT_RUNTIME_CONFIG, "runtime-config"],
385+
[CLOUD_ENDPOINT_CONFIRMED_ORIGIN, "confirmed-origin"],
382386
[RELAY_URL_SECRET, "https://relay.example.test"],
383387
[CLOUD_CLI_DESIRED_LINK_SECRET, "managed"],
384388
] as const;
385389

390+
it.effect("does not recreate a link that was unlinked while startup registration retried", () => {
391+
const { store, values } = makeMemorySecretStore(managedLinkSecrets);
392+
const applyConfigCalls: Array<unknown> = [];
393+
const requests: Array<HttpClientRequest.HttpClientRequest> = [];
394+
395+
return Effect.gen(function* () {
396+
// Registration started while this marker existed. Unlink removes it
397+
// before startup receives the relay's final not_linked response.
398+
values.delete(CLOUD_CLI_DESIRED_LINK_SECRET);
399+
400+
expect(yield* reconcileDesiredCloudLinkIfStillDesired("http://127.0.0.1:3773")).toBeNull();
401+
expect(requests).toEqual([]);
402+
expect(applyConfigCalls).toEqual([]);
403+
}).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
404+
});
405+
386406
it.effect("stops the connector, releases the relay tunnel, and drops the dead token", () => {
387407
const { store, values } = makeMemorySecretStore(managedLinkSecrets);
388408
const applyConfigCalls: Array<unknown> = [];
@@ -401,6 +421,7 @@ describe("releaseManagedTunnelOnShutdown", () => {
401421
);
402422
expect(request.headers.authorization).toBe("Bearer cli-access-token");
403423
expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(false);
424+
expect(values.has(CLOUD_ENDPOINT_CONFIRMED_ORIGIN)).toBe(false);
404425
}).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
405426
});
406427

@@ -611,7 +632,7 @@ describe("releaseManagedTunnelOnShutdown", () => {
611632
);
612633
});
613634

614-
it.effect("registers an existing tunnel without provisioning or restarting it", () => {
635+
it.effect("registers an existing tunnel and starts the confirmed connector", () => {
615636
const { store } = makeMemorySecretStore([
616637
[
617638
CLOUD_ENDPOINT_RUNTIME_CONFIG,
@@ -625,7 +646,9 @@ describe("releaseManagedTunnelOnShutdown", () => {
625646
const requests: Array<HttpClientRequest.HttpClientRequest> = [];
626647

627648
return Effect.gen(function* () {
628-
expect(yield* registerManagedCloudTunnelRecovery()).toBe(true);
649+
expect(yield* registerManagedCloudTunnelRecovery("http://127.0.0.1:3773")).toMatchObject({
650+
status: "ready",
651+
});
629652
expect(requests).toHaveLength(1);
630653
expect(requests[0]?.method).toBe("POST");
631654
expect(requests[0]?.url).toBe(
@@ -640,12 +663,118 @@ describe("releaseManagedTunnelOnShutdown", () => {
640663
).toMatchObject({
641664
cloudUserId: "user-123",
642665
tunnelId: "existing-tunnel",
666+
origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
643667
});
644668
}
669+
expect(applyConfigCalls).toHaveLength(1);
670+
}).pipe(
671+
provideReleaseHarness({
672+
store,
673+
applyConfigCalls,
674+
requests,
675+
respond: () => Response.json({ status: "ready" }),
676+
}),
677+
);
678+
});
679+
680+
it.effect(
681+
"starts a connector with a marker for the current origin without contacting relay",
682+
() => {
683+
const configJson =
684+
'{"providerKind":"cloudflare_tunnel","connectorToken":"existing-token","tunnelId":"existing-tunnel"}';
685+
const config = {
686+
providerKind: "cloudflare_tunnel" as const,
687+
connectorToken: "existing-token",
688+
tunnelId: "existing-tunnel",
689+
};
690+
const { store } = makeMemorySecretStore([
691+
[CLOUD_ENDPOINT_RUNTIME_CONFIG, configJson],
692+
[
693+
CLOUD_ENDPOINT_CONFIRMED_ORIGIN,
694+
`{"config":${configJson},"origin":{"localHttpHost":"127.0.0.1","localHttpPort":3773}}`,
695+
],
696+
]);
697+
const applyConfigCalls: Array<unknown> = [];
698+
const requests: Array<HttpClientRequest.HttpClientRequest> = [];
699+
700+
return Effect.gen(function* () {
701+
expect(yield* startManagedCloudTunnelIfOriginConfirmed("http://127.0.0.1:3773")).toBe(true);
702+
expect(applyConfigCalls).toEqual([config]);
703+
expect(requests).toEqual([]);
704+
}).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
705+
},
706+
);
707+
708+
it.effect.each([
709+
{ name: "missing", marker: undefined, origin: "http://127.0.0.1:3773" },
710+
{
711+
name: "stale",
712+
marker:
713+
'{"config":{"providerKind":"cloudflare_tunnel","connectorToken":"existing-token","tunnelId":"existing-tunnel"},"origin":{"localHttpHost":"127.0.0.1","localHttpPort":3773}}',
714+
origin: "http://127.0.0.1:4884",
715+
},
716+
])("does not start a connector with a $name origin marker", ({ marker, origin }) => {
717+
const entries: Array<readonly [string, string]> = [
718+
[
719+
CLOUD_ENDPOINT_RUNTIME_CONFIG,
720+
'{"providerKind":"cloudflare_tunnel","connectorToken":"existing-token","tunnelId":"existing-tunnel"}',
721+
],
722+
];
723+
if (marker !== undefined) entries.push([CLOUD_ENDPOINT_CONFIRMED_ORIGIN, marker]);
724+
const { store } = makeMemorySecretStore(entries);
725+
const applyConfigCalls: Array<unknown> = [];
726+
const requests: Array<HttpClientRequest.HttpClientRequest> = [];
727+
728+
return Effect.gen(function* () {
729+
expect(yield* startManagedCloudTunnelIfOriginConfirmed(origin)).toBe(false);
645730
expect(applyConfigCalls).toEqual([]);
731+
expect(requests).toEqual([]);
646732
}).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
647733
});
648734

735+
it.effect.each(["replaced", "removed"] as const)(
736+
"does not activate a tunnel when its runtime config is %s during registration",
737+
(mutation) => {
738+
const originalConfig =
739+
'{"providerKind":"cloudflare_tunnel","connectorToken":"existing-token","tunnelId":"existing-tunnel"}';
740+
const { store, values } = makeMemorySecretStore([
741+
[CLOUD_ENDPOINT_RUNTIME_CONFIG, originalConfig],
742+
[RELAY_URL_SECRET, "https://relay.example.test"],
743+
[CLOUD_LINKED_USER_ID, "user-123"],
744+
[RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"],
745+
]);
746+
const applyConfigCalls: Array<unknown> = [];
747+
const requests: Array<HttpClientRequest.HttpClientRequest> = [];
748+
749+
return Effect.gen(function* () {
750+
expect(yield* registerManagedCloudTunnelRecovery("http://127.0.0.1:3773")).toEqual({
751+
status: "superseded",
752+
});
753+
expect(applyConfigCalls).toEqual([]);
754+
expect(values.has(CLOUD_ENDPOINT_CONFIRMED_ORIGIN)).toBe(false);
755+
}).pipe(
756+
provideReleaseHarness({
757+
store,
758+
applyConfigCalls,
759+
requests,
760+
respond: () => {
761+
if (mutation === "replaced") {
762+
values.set(
763+
CLOUD_ENDPOINT_RUNTIME_CONFIG,
764+
new TextEncoder().encode(
765+
'{"providerKind":"cloudflare_tunnel","connectorToken":"fresh-token","tunnelId":"fresh-tunnel"}',
766+
),
767+
);
768+
} else {
769+
values.delete(CLOUD_ENDPOINT_RUNTIME_CONFIG);
770+
}
771+
return Response.json({ status: "ready" });
772+
},
773+
}),
774+
);
775+
},
776+
);
777+
649778
it.effect("does not register recovery without a recorded tunnel ID", () => {
650779
const { store } = makeMemorySecretStore([
651780
[
@@ -660,7 +789,9 @@ describe("releaseManagedTunnelOnShutdown", () => {
660789
const requests: Array<HttpClientRequest.HttpClientRequest> = [];
661790

662791
return Effect.gen(function* () {
663-
expect(yield* registerManagedCloudTunnelRecovery()).toBe(false);
792+
expect(yield* registerManagedCloudTunnelRecovery("http://127.0.0.1:3773")).toEqual({
793+
status: "not_linked",
794+
});
664795
expect(requests).toEqual([]);
665796
expect(applyConfigCalls).toEqual([]);
666797
}).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));

0 commit comments

Comments
 (0)