Skip to content

Commit cd8a986

Browse files
fix(server): T3 Connect backs off from a failing relay instead of hammering it (pingdotgg#22108)
Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent b1167b6 commit cd8a986

5 files changed

Lines changed: 455 additions & 71 deletions

File tree

‎apps/server/src/cloud/managedTunnelStartup.test.ts‎

Lines changed: 206 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,21 @@
11
import { describe, expect, it } from "@effect/vitest";
2+
import * as Clock from "effect/Clock";
23
import * as Deferred from "effect/Deferred";
4+
import * as Duration from "effect/Duration";
35
import * as Effect from "effect/Effect";
46
import * as Fiber from "effect/Fiber";
57
import * as TestClock from "effect/testing/TestClock";
68

79
import {
10+
MANAGED_TUNNEL_RECOVERY_COOLDOWN,
11+
makeManagedTunnelRecovery,
812
managedTunnelStartupAction,
13+
nextManagedTunnelRecoveryCooldown,
14+
relayRetrySchedule,
15+
retryDesiredLinkReconcile,
916
retryManagedTunnelRegistration,
1017
} from "./managedTunnelStartup.ts";
18+
import { RelayRequestError } from "./relayResponse.ts";
1119

1220
describe("managedTunnelStartupAction", () => {
1321
const config = {
@@ -129,3 +137,201 @@ describe("retryManagedTunnelRegistration", () => {
129137
}),
130138
);
131139
});
140+
141+
describe("nextManagedTunnelRecoveryCooldown", () => {
142+
const minutes = (value: number) => Duration.minutes(value);
143+
144+
it("backs off while recoveries keep failing to fix the tunnel, up to 30 minutes", () => {
145+
let cooldown = MANAGED_TUNNEL_RECOVERY_COOLDOWN;
146+
const seen: Array<number> = [];
147+
for (let attempt = 0; attempt < 6; attempt++) {
148+
cooldown = nextManagedTunnelRecoveryCooldown({
149+
current: cooldown,
150+
sinceLastRecovery: minutes(3),
151+
});
152+
seen.push(Duration.toMinutes(cooldown));
153+
}
154+
expect(seen).toEqual([4, 8, 16, 30, 30, 30]);
155+
});
156+
157+
it("starts over once the previous recovery held", () => {
158+
expect(
159+
Duration.toMinutes(
160+
nextManagedTunnelRecoveryCooldown({ current: minutes(16), sinceLastRecovery: minutes(26) }),
161+
),
162+
).toBe(2);
163+
expect(
164+
Duration.toMinutes(
165+
nextManagedTunnelRecoveryCooldown({ current: minutes(16), sinceLastRecovery: minutes(20) }),
166+
),
167+
).toBe(30);
168+
});
169+
});
170+
171+
describe("relayRetrySchedule", () => {
172+
it.effect("waits at least as long as the relay's Retry-After", () =>
173+
Effect.gen(function* () {
174+
let attempts = 0;
175+
const request = Effect.suspend(() => {
176+
attempts += 1;
177+
return attempts === 1
178+
? Effect.fail(
179+
new RelayRequestError({
180+
rejection: "unavailable",
181+
description: "paused",
182+
retryAfterMillis: 120_000,
183+
}),
184+
)
185+
: Effect.succeed("done");
186+
});
187+
const fiber = yield* Effect.forkChild(
188+
Effect.retry(request, { schedule: relayRetrySchedule }),
189+
{
190+
startImmediately: true,
191+
},
192+
);
193+
yield* TestClock.adjust("90 seconds");
194+
expect(attempts).toBe(1);
195+
yield* TestClock.adjust("31 seconds");
196+
expect(yield* Fiber.join(fiber)).toBe("done");
197+
expect(attempts).toBe(2);
198+
}),
199+
);
200+
});
201+
202+
describe("makeManagedTunnelRecovery", () => {
203+
const config = {
204+
providerKind: "cloudflare_tunnel" as const,
205+
connectorToken: "connector-token",
206+
tunnelId: "tunnel-1",
207+
};
208+
209+
it.effect("retries an unavailable relay every cooldown, and cools down from when it ends", () =>
210+
Effect.gen(function* () {
211+
const attemptMinutes: Array<number> = [];
212+
const recover = makeManagedTunnelRecovery({
213+
recover: () =>
214+
Effect.gen(function* () {
215+
attemptMinutes.push((yield* Clock.currentTimeMillis) / 60_000);
216+
return attemptMinutes.length < 4
217+
? yield* Effect.fail("relay unavailable" as const)
218+
: true;
219+
}),
220+
isRetryable: () => true,
221+
});
222+
const run = yield* recover;
223+
const fiber = yield* Effect.forkChild(run(config), { startImmediately: true });
224+
yield* TestClock.adjust("6 minutes");
225+
expect(yield* Fiber.join(fiber)).toBe(true);
226+
expect(attemptMinutes.map(Math.round)).toEqual([0, 2, 4, 6]);
227+
228+
yield* TestClock.adjust("1 minute");
229+
const next = yield* Effect.forkChild(run(config), { startImmediately: true });
230+
yield* TestClock.adjust("10 minutes");
231+
yield* Fiber.join(next);
232+
expect(attemptMinutes.map(Math.round)).toEqual([0, 2, 4, 6, 10]);
233+
}),
234+
);
235+
});
236+
237+
describe("retryManagedTunnelRegistration fallback", () => {
238+
it.effect("falls back by the end of the window and keeps honoring Retry-After", () =>
239+
Effect.gen(function* () {
240+
let fallbackAtMillis: number | undefined;
241+
const attemptMillis: Array<number> = [];
242+
const registration = Clock.currentTimeMillis.pipe(
243+
Effect.tap((now) => Effect.sync(() => attemptMillis.push(now))),
244+
Effect.andThen(
245+
Effect.fail(
246+
new RelayRequestError({
247+
rejection: "unavailable",
248+
description: "paused",
249+
retryAfterMillis: 900_000,
250+
}),
251+
),
252+
),
253+
);
254+
yield* Effect.forkChild(
255+
retryManagedTunnelRegistration(
256+
registration,
257+
() => true,
258+
Clock.currentTimeMillis.pipe(
259+
Effect.tap((now) =>
260+
Effect.sync(() => {
261+
fallbackAtMillis ??= now;
262+
}),
263+
),
264+
Effect.asVoid,
265+
),
266+
),
267+
{ startImmediately: true },
268+
);
269+
yield* TestClock.adjust("1 hour");
270+
expect(fallbackAtMillis).toBeDefined();
271+
expect(fallbackAtMillis).toBeLessThanOrEqual(10 * 60_000);
272+
expect(attemptMillis.length).toBeGreaterThan(2);
273+
for (let index = 1; index < attemptMillis.length; index++) {
274+
expect(attemptMillis[index]! - attemptMillis[index - 1]!).toBeGreaterThanOrEqual(900_000);
275+
}
276+
}),
277+
);
278+
});
279+
280+
describe("retryDesiredLinkReconcile", () => {
281+
const relayUnavailable = new RelayRequestError({
282+
rejection: "unavailable",
283+
description: "relay unavailable",
284+
});
285+
286+
it.effect("gives up on a local failure once the retry window ends", () =>
287+
Effect.gen(function* () {
288+
let attempts = 0;
289+
let windowLogs = 0;
290+
const fiber = yield* Effect.forkChild(
291+
Effect.flip(
292+
retryDesiredLinkReconcile(
293+
Effect.suspend(() => {
294+
attempts += 1;
295+
return Effect.fail("cloudflared cannot run" as const);
296+
}),
297+
() => true,
298+
() =>
299+
Effect.sync(() => {
300+
windowLogs += 1;
301+
}),
302+
),
303+
),
304+
{ startImmediately: true },
305+
);
306+
yield* TestClock.adjust("2 hours");
307+
expect(yield* Fiber.join(fiber)).toBe("cloudflared cannot run");
308+
expect(attempts).toBeLessThan(20);
309+
expect(windowLogs).toBe(0);
310+
}),
311+
);
312+
313+
it.effect("keeps relinking while the relay is down and succeeds when it returns", () =>
314+
Effect.gen(function* () {
315+
let windowLogs = 0;
316+
const reconcile = Effect.gen(function* () {
317+
return (yield* Clock.currentTimeMillis) < 30 * 60_000
318+
? yield* Effect.fail(relayUnavailable)
319+
: "managed";
320+
});
321+
const fiber = yield* Effect.forkChild(
322+
retryDesiredLinkReconcile(
323+
reconcile,
324+
() => true,
325+
() =>
326+
Effect.sync(() => {
327+
windowLogs += 1;
328+
}),
329+
),
330+
{ startImmediately: true },
331+
);
332+
yield* TestClock.adjust("45 minutes");
333+
expect(yield* Fiber.join(fiber)).toBe("managed");
334+
expect(windowLogs).toBe(1);
335+
}),
336+
);
337+
});

0 commit comments

Comments
 (0)