Skip to content

Commit 2aa5f83

Browse files
t3dotggclaude
andcommitted
fix: address review findings on connection promotion
- Track promotion cooldowns per endpoint so a second failed candidate does not clear the first one's cooldown (Bugbot, Macroscope). - Probe a direct route's descriptor with the 3s cached-endpoint timeout so a black-holed LAN address leaves the 15s establish budget for the relay fallback (Bugbot). - Advertise the loopback endpoint at the host the server actually binds (`::1`, `127.0.0.2`) instead of assuming 127.0.0.1, and withhold the Tailscale Serve endpoint when Serve cannot reach that binding (Macroscope). - Tag promoted direct sockets with connectionMethod=direct for the analytics field main added. Co-Authored-By: Claude Fable 5.1 <[email protected]>
1 parent f1d970a commit 2aa5f83

6 files changed

Lines changed: 229 additions & 20 deletions

File tree

‎apps/server/src/remoteAccess/ServerAdvertisedEndpoints.test.ts‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -126,4 +126,20 @@ describe("resolveServerAdvertisedEndpoints", () => {
126126
[["server-private-network:http://100.100.100.100:3773", "private-network"]],
127127
);
128128
});
129+
130+
it("echoes an explicit IPv6 loopback binding instead of assuming 127.0.0.1", () => {
131+
const endpoints = resolveServerAdvertisedEndpoints({
132+
port: 3773,
133+
host: "::1",
134+
networkInterfaces: {},
135+
tailscaleEndpoints: [tailscaleServeEndpoint],
136+
});
137+
138+
// Tailscale Serve only forwards to 127.0.0.1, which this binding does not
139+
// listen on, so the HTTPS endpoint is withheld too.
140+
assert.deepEqual(
141+
endpoints.map((endpoint) => endpoint.httpBaseUrl),
142+
["http://[::1]:3773/"],
143+
);
144+
});
129145
});

‎apps/server/src/remoteAccess/ServerAdvertisedEndpoints.ts‎

Lines changed: 22 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -55,11 +55,22 @@ const resolveBindingMode = (host: string | undefined): BindingMode => {
5555
return "specific";
5656
};
5757

58-
/** Tailscale Serve proxies to 127.0.0.1, so its HTTPS endpoint stays reachable
59-
* under a loopback binding; the plain Tailnet-IP HTTP endpoints do not. */
58+
/** Tailscale Serve proxies to 127.0.0.1 (see `ensureTailscaleServe`), so its
59+
* HTTPS endpoint stays reachable under a loopback binding; the plain
60+
* Tailnet-IP HTTP endpoints do not. */
6061
const isHttpsEndpoint = (endpoint: AdvertisedEndpoint): boolean =>
6162
endpoint.httpBaseUrl.startsWith("https://");
6263

64+
/** The loopback address the server actually listens on. An unset host binds
65+
* 127.0.0.1 (see server.ts); an explicit `::1` or `127.0.0.2` must be echoed
66+
* back or the advertised URL points at an address nobody is listening on. */
67+
const resolveLoopbackHost = (host: string | undefined): string =>
68+
!host || host === "localhost" ? "127.0.0.1" : host;
69+
70+
/** Serve only forwards to 127.0.0.1, so it can reach the server only when the
71+
* server listens there. */
72+
const isTailscaleServeReachable = (loopbackHost: string): boolean => loopbackHost === "127.0.0.1";
73+
6374
export interface ResolveServerAdvertisedEndpointsInput {
6475
readonly port: number;
6576
readonly host: string | undefined;
@@ -74,19 +85,22 @@ export function resolveServerAdvertisedEndpoints(
7485
input: ResolveServerAdvertisedEndpointsInput,
7586
): readonly AdvertisedEndpoint[] {
7687
const mode = resolveBindingMode(input.host);
88+
const loopbackHost = mode === "loopback" ? resolveLoopbackHost(input.host) : "127.0.0.1";
7789
const loopbackEndpoint = createAdvertisedEndpoint({
7890
provider: SERVER_ENDPOINT_PROVIDER,
7991
source: "server",
8092
id: `server-loopback:${input.port}`,
8193
label: "This machine",
82-
httpBaseUrl: `http://127.0.0.1:${input.port}`,
94+
httpBaseUrl: `http://${formatHostForUrl(loopbackHost)}:${input.port}`,
8395
reachability: "loopback",
8496
status: "available",
8597
description: "Loopback endpoint for this server.",
8698
});
8799

88100
if (mode === "loopback") {
89-
return [loopbackEndpoint, ...input.tailscaleEndpoints.filter(isHttpsEndpoint)];
101+
return isTailscaleServeReachable(loopbackHost)
102+
? [loopbackEndpoint, ...input.tailscaleEndpoints.filter(isHttpsEndpoint)]
103+
: [loopbackEndpoint];
90104
}
91105

92106
if (mode === "specific") {
@@ -179,7 +193,10 @@ export const make = Effect.gen(function* () {
179193
// Skip the Tailscale resolver when nothing Tailscale-reachable could exist:
180194
// spawning the CLI triggers a macOS TCC prompt on sandboxed installs.
181195
const shouldResolveTailscale =
182-
mode === "wildcard" || (mode === "loopback" && config.tailscaleServeEnabled);
196+
mode === "wildcard" ||
197+
(mode === "loopback" &&
198+
config.tailscaleServeEnabled &&
199+
isTailscaleServeReachable(resolveLoopbackHost(config.host)));
183200
const tailscaleEndpoints = shouldResolveTailscale
184201
? yield* resolveTailscaleAdvertisedEndpoints({
185202
port,

‎packages/client-runtime/src/authorization/layer.test.ts‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { AuthStandardClientScopes, EnvironmentId } from "@t3tools/contracts";
22
import { describe, expect, it } from "@effect/vitest";
33
import * as Effect from "effect/Effect";
4+
import * as Fiber from "effect/Fiber";
45
import * as Layer from "effect/Layer";
56
import * as Option from "effect/Option";
67
import * as Ref from "effect/Ref";
@@ -149,6 +150,7 @@ const makeHarness = Effect.fn("TestRemoteAuthorization.makeHarness")(function* (
149150
return {
150151
layer,
151152
tokens,
153+
tokenStore,
152154
bootstrapCalls,
153155
proofInputs,
154156
fetch,
@@ -407,6 +409,66 @@ describe("RemoteEnvironmentAuthorization", () => {
407409
}),
408410
);
409411

412+
it.effect(
413+
"times out a black-holed direct route quickly so the relay fallback keeps its budget",
414+
() =>
415+
Effect.gen(function* () {
416+
const cached = new TokenStore.RemoteDpopAccessToken({
417+
environmentId: ENVIRONMENT_ID,
418+
label: DESCRIPTOR.label,
419+
endpoint: ENDPOINT,
420+
accessToken: "cached-access-token",
421+
expiresAtEpochMs: Number.MAX_SAFE_INTEGER,
422+
dpopThumbprint: "thumbprint-1",
423+
});
424+
// A LAN address that accepts the TCP connection but never answers.
425+
const hanging = new Promise<Response>(() => {});
426+
const harness = yield* makeHarness({ initialToken: cached, responses: [] });
427+
const layer = RemoteEnvironmentAuthorization.layer.pipe(
428+
Layer.provide(
429+
Layer.mergeAll(
430+
remoteHttpClientLayer(() => hanging),
431+
Layer.succeed(
432+
ManagedRelay.ManagedRelayDpopSigner,
433+
ManagedRelay.ManagedRelayDpopSigner.of({
434+
thumbprint: Effect.succeed("thumbprint-1"),
435+
createProof: (proofInput) => Effect.succeed(`proof:${proofInput.url}`),
436+
}),
437+
),
438+
Layer.succeed(TokenStore.RemoteDpopAccessTokenStore, harness.tokenStore),
439+
Layer.succeed(
440+
ClientCapabilities.ClientPresentation,
441+
ClientCapabilities.ClientPresentation.of({
442+
metadata: { label: "T3 Code Test", deviceType: "mobile", os: "test" },
443+
scopes: AuthStandardClientScopes,
444+
}),
445+
),
446+
),
447+
),
448+
);
449+
450+
const attempt = yield* Effect.gen(function* () {
451+
const remote = yield* RemoteEnvironmentAuthorization.RemoteEnvironmentAuthorization;
452+
return yield* remote.authorizeDpopDirect({
453+
expectedEnvironmentId: ENVIRONMENT_ID,
454+
endpoint: {
455+
httpBaseUrl: "http://192.168.1.20:3773",
456+
wsBaseUrl: "ws://192.168.1.20:3773",
457+
},
458+
});
459+
}).pipe(Effect.provide(layer), Effect.result, Effect.forkChild);
460+
461+
// The supervisor's whole establish attempt is capped at 15 seconds, so
462+
// the direct probe must give up well inside that, not at the default 10s.
463+
yield* TestClock.adjust("3 seconds");
464+
const result = yield* Fiber.join(attempt);
465+
expect(result._tag).toBe("Failure");
466+
if (result._tag === "Failure") {
467+
expect(result.failure._tag).toBe("ConnectionTransientError");
468+
}
469+
}),
470+
);
471+
410472
it.effect("does not persist a refreshed token until its websocket ticket succeeds", () =>
411473
Effect.gen(function* () {
412474
const harness = yield* makeHarness({

‎packages/client-runtime/src/authorization/service.ts‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -370,8 +370,14 @@ export const make = Effect.gen(function* () {
370370
});
371371
}
372372
// The descriptor check keeps us from talking to an unrelated server that
373-
// answers on the direct address.
374-
const descriptor = yield* fetchDescriptor(input.endpoint.httpBaseUrl).pipe(
373+
// answers on the direct address. It uses the short cached-endpoint
374+
// timeout so a black-holed LAN address leaves the establish budget
375+
// (15s in the supervisor) mostly intact for the relay fallback.
376+
const descriptor = yield* fetchRemoteEnvironmentDescriptor({
377+
httpBaseUrl: input.endpoint.httpBaseUrl,
378+
timeoutMs: CACHED_ENDPOINT_SOCKET_TIMEOUT_MS,
379+
}).pipe(
380+
Effect.mapError(mapRemoteEnvironmentError),
375381
Effect.provideService(HttpClient.HttpClient, httpClient),
376382
);
377383
if (descriptor.environmentId !== input.expectedEnvironmentId) {

‎packages/client-runtime/src/connection/promotion.test.ts‎

Lines changed: 99 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,14 @@
1-
import type { AdvertisedEndpoint } from "@t3tools/contracts";
1+
import { type AdvertisedEndpoint, EnvironmentId } from "@t3tools/contracts";
22
import { describe, expect, it } from "@effect/vitest";
3+
import * as Effect from "effect/Effect";
4+
import * as Layer from "effect/Layer";
5+
import * as Option from "effect/Option";
6+
import * as TestClock from "effect/testing/TestClock";
37

8+
import * as ManagedRelay from "../relay/managedRelay.ts";
9+
import { remoteHttpClientLayer } from "../rpc/http.ts";
10+
import type { PreparedConnection } from "./model.ts";
11+
import * as ConnectionPromotion from "./promotion.ts";
412
import { selectPromotionCandidates } from "./promotion.ts";
513

614
function endpoint(input: {
@@ -94,3 +102,93 @@ describe("selectPromotionCandidates", () => {
94102
expect(candidates).toEqual([]);
95103
});
96104
});
105+
106+
const ENVIRONMENT_ID = EnvironmentId.make("environment-1");
107+
const LAN_ENDPOINT = endpoint({
108+
id: "server-lan:http://192.168.1.20:3773",
109+
httpBaseUrl: "http://192.168.1.20:3773/",
110+
reachability: "lan",
111+
});
112+
const TAILNET_ENDPOINT = endpoint({
113+
id: "tailscale-ip:http://100.100.100.100:3773",
114+
httpBaseUrl: "http://100.100.100.100:3773/",
115+
reachability: "private-network",
116+
});
117+
const RELAY_PREPARED: PreparedConnection = {
118+
environmentId: ENVIRONMENT_ID,
119+
label: "Remote",
120+
httpBaseUrl: RELAY_BASE_URL,
121+
socketUrl: "wss://tunnel.example.test/ws",
122+
httpAuthorization: { _tag: "Dpop", accessToken: "access-token" },
123+
target: { _tag: "RelayConnectionTarget", environmentId: ENVIRONMENT_ID, label: "Remote" },
124+
};
125+
126+
// Fake fetch: the relay origin answers the advertised endpoint list, and every
127+
// direct candidate answers the descriptor probe as the same environment.
128+
const promotionHarnessLayer = (endpoints: readonly AdvertisedEndpoint[]) => {
129+
const fetchFn = ((input) => {
130+
const url = String(input instanceof Request ? input.url : input);
131+
if (url.endsWith("/api/remote-access/endpoints")) {
132+
return Promise.resolve(Response.json(endpoints));
133+
}
134+
if (url.endsWith("/.well-known/t3/environment")) {
135+
return Promise.resolve(
136+
Response.json({
137+
environmentId: ENVIRONMENT_ID,
138+
label: "Remote",
139+
platform: { os: "linux", arch: "x64" },
140+
serverVersion: "0.0.0-test",
141+
capabilities: { repositoryIdentity: true },
142+
}),
143+
);
144+
}
145+
return Promise.reject(new Error(`Unexpected fetch call to ${url}`));
146+
}) satisfies typeof fetch;
147+
return ConnectionPromotion.layer.pipe(
148+
Layer.provide(
149+
Layer.mergeAll(
150+
remoteHttpClientLayer(fetchFn),
151+
Layer.succeed(
152+
ManagedRelay.ManagedRelayDpopSigner,
153+
ManagedRelay.ManagedRelayDpopSigner.of({
154+
thumbprint: Effect.succeed("thumbprint-1"),
155+
createProof: (proofInput) => Effect.succeed(`proof:${proofInput.url}`),
156+
}),
157+
),
158+
),
159+
),
160+
);
161+
};
162+
163+
describe("ConnectionPromotion cooldowns", () => {
164+
it.effect("keeps every failed endpoint cooling down, not just the latest one", () =>
165+
Effect.gen(function* () {
166+
const promotion = yield* ConnectionPromotion.ConnectionPromotion;
167+
168+
// LAN wins first (ranked above the tailnet), then fails.
169+
const first = yield* promotion.discover(RELAY_PREPARED);
170+
expect(Option.map(first, (route) => route.endpointId)).toEqual(Option.some(LAN_ENDPOINT.id));
171+
yield* promotion.reportOverrideFailed(ENVIRONMENT_ID);
172+
173+
// The tailnet route is promoted next and fails within the cooldown.
174+
const second = yield* promotion.discover(RELAY_PREPARED);
175+
expect(Option.map(second, (route) => route.endpointId)).toEqual(
176+
Option.some(TAILNET_ENDPOINT.id),
177+
);
178+
yield* TestClock.adjust("1 minute");
179+
yield* promotion.reportOverrideFailed(ENVIRONMENT_ID);
180+
181+
// Both cooldowns are still active: the LAN route must not be re-selected
182+
// just because the tailnet failure came later.
183+
expect(yield* promotion.discover(RELAY_PREPARED)).toEqual(Option.none());
184+
185+
// The LAN cooldown started first, so it expires first.
186+
yield* TestClock.adjust("4 minutes");
187+
const third = yield* promotion.discover(RELAY_PREPARED);
188+
expect(Option.map(third, (route) => route.endpointId)).toEqual(Option.some(LAN_ENDPOINT.id));
189+
}).pipe(
190+
Effect.provide(promotionHarnessLayer([LAN_ENDPOINT, TAILNET_ENDPOINT])),
191+
Effect.provide(TestClock.layer()),
192+
),
193+
);
194+
});

‎packages/client-runtime/src/connection/promotion.ts‎

Lines changed: 22 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -77,10 +77,9 @@ export function selectPromotionCandidates(input: {
7777
.slice(0, MAX_PROBED_CANDIDATES);
7878
}
7979

80-
interface PromotionCooldown {
81-
readonly endpointId: string;
82-
readonly failedAtEpochMs: number;
83-
}
80+
/** Failure time per endpoint id. Keyed per endpoint (not per environment) so
81+
* two flaky routes cannot clear each other's cooldown. */
82+
type PromotionCooldowns = ReadonlyMap<string, number>;
8483

8584
/**
8685
* Discovers direct routes for relay-connected environments and remembers the
@@ -132,7 +131,7 @@ export const make = Effect.gen(function* () {
132131
const signer = yield* ManagedRelay.ManagedRelayDpopSigner;
133132
const httpClient = yield* HttpClient.HttpClient;
134133
const overrides = yield* Ref.make<ReadonlyMap<EnvironmentId, PromotedRoute>>(new Map());
135-
const cooldowns = yield* Ref.make<ReadonlyMap<EnvironmentId, PromotionCooldown>>(new Map());
134+
const cooldowns = yield* Ref.make<ReadonlyMap<EnvironmentId, PromotionCooldowns>>(new Map());
136135

137136
const overrideFor = Effect.fn("ConnectionPromotion.overrideFor")(function* (
138137
environmentId: EnvironmentId,
@@ -156,7 +155,9 @@ export const make = Effect.gen(function* () {
156155
});
157156
yield* Ref.update(cooldowns, (current) => {
158157
const next = new Map(current);
159-
next.set(environmentId, { endpointId: override.endpointId, failedAtEpochMs: now });
158+
const forEnvironment = new Map(current.get(environmentId) ?? []);
159+
forEnvironment.set(override.endpointId, now);
160+
next.set(environmentId, forEnvironment);
160161
return next;
161162
});
162163
yield* Effect.logInfo("Direct environment route failed; falling back to the relay.").pipe(
@@ -167,21 +168,30 @@ export const make = Effect.gen(function* () {
167168
);
168169
});
169170

171+
// Returns the endpoint ids still cooling down and prunes expired entries.
170172
const activeCooldownEndpointIds = Effect.fnUntraced(function* (environmentId: EnvironmentId) {
171-
const cooldown = (yield* Ref.get(cooldowns)).get(environmentId);
172-
if (cooldown === undefined) {
173+
const forEnvironment = (yield* Ref.get(cooldowns)).get(environmentId);
174+
if (forEnvironment === undefined) {
173175
return new Set<string>();
174176
}
175177
const now = yield* Clock.currentTimeMillis;
176-
if (cooldown.failedAtEpochMs + PROMOTION_FAILURE_COOLDOWN_MS <= now) {
178+
const active = new Map(
179+
[...forEnvironment].filter(
180+
([, failedAtEpochMs]) => failedAtEpochMs + PROMOTION_FAILURE_COOLDOWN_MS > now,
181+
),
182+
);
183+
if (active.size !== forEnvironment.size) {
177184
yield* Ref.update(cooldowns, (current) => {
178185
const next = new Map(current);
179-
next.delete(environmentId);
186+
if (active.size === 0) {
187+
next.delete(environmentId);
188+
} else {
189+
next.set(environmentId, active);
190+
}
180191
return next;
181192
});
182-
return new Set<string>();
183193
}
184-
return new Set([cooldown.endpointId]);
194+
return new Set(active.keys());
185195
});
186196

187197
const probeCandidate = Effect.fnUntraced(function* (

0 commit comments

Comments
 (0)