|
1 | 1 | import { describe, expect, it } from "@effect/vitest"; |
| 2 | +import * as Clock from "effect/Clock"; |
2 | 3 | import * as Deferred from "effect/Deferred"; |
| 4 | +import * as Duration from "effect/Duration"; |
3 | 5 | import * as Effect from "effect/Effect"; |
4 | 6 | import * as Fiber from "effect/Fiber"; |
5 | 7 | import * as TestClock from "effect/testing/TestClock"; |
6 | 8 |
|
7 | 9 | import { |
| 10 | + MANAGED_TUNNEL_RECOVERY_COOLDOWN, |
| 11 | + makeManagedTunnelRecovery, |
8 | 12 | managedTunnelStartupAction, |
| 13 | + nextManagedTunnelRecoveryCooldown, |
| 14 | + relayRetrySchedule, |
| 15 | + retryDesiredLinkReconcile, |
9 | 16 | retryManagedTunnelRegistration, |
10 | 17 | } from "./managedTunnelStartup.ts"; |
| 18 | +import { RelayRequestError } from "./relayResponse.ts"; |
11 | 19 |
|
12 | 20 | describe("managedTunnelStartupAction", () => { |
13 | 21 | const config = { |
@@ -129,3 +137,201 @@ describe("retryManagedTunnelRegistration", () => { |
129 | 137 | }), |
130 | 138 | ); |
131 | 139 | }); |
| 140 | + |
| 141 | +describe("nextManagedTunnelRecoveryCooldown", () => { |
| 142 | + const minutes = (value: number) => Duration.minutes(value); |
| 143 | + |
| 144 | + it("backs off while recoveries keep failing to fix the tunnel, up to 30 minutes", () => { |
| 145 | + let cooldown = MANAGED_TUNNEL_RECOVERY_COOLDOWN; |
| 146 | + const seen: Array<number> = []; |
| 147 | + for (let attempt = 0; attempt < 6; attempt++) { |
| 148 | + cooldown = nextManagedTunnelRecoveryCooldown({ |
| 149 | + current: cooldown, |
| 150 | + sinceLastRecovery: minutes(3), |
| 151 | + }); |
| 152 | + seen.push(Duration.toMinutes(cooldown)); |
| 153 | + } |
| 154 | + expect(seen).toEqual([4, 8, 16, 30, 30, 30]); |
| 155 | + }); |
| 156 | + |
| 157 | + it("starts over once the previous recovery held", () => { |
| 158 | + expect( |
| 159 | + Duration.toMinutes( |
| 160 | + nextManagedTunnelRecoveryCooldown({ current: minutes(16), sinceLastRecovery: minutes(26) }), |
| 161 | + ), |
| 162 | + ).toBe(2); |
| 163 | + expect( |
| 164 | + Duration.toMinutes( |
| 165 | + nextManagedTunnelRecoveryCooldown({ current: minutes(16), sinceLastRecovery: minutes(20) }), |
| 166 | + ), |
| 167 | + ).toBe(30); |
| 168 | + }); |
| 169 | +}); |
| 170 | + |
| 171 | +describe("relayRetrySchedule", () => { |
| 172 | + it.effect("waits at least as long as the relay's Retry-After", () => |
| 173 | + Effect.gen(function* () { |
| 174 | + let attempts = 0; |
| 175 | + const request = Effect.suspend(() => { |
| 176 | + attempts += 1; |
| 177 | + return attempts === 1 |
| 178 | + ? Effect.fail( |
| 179 | + new RelayRequestError({ |
| 180 | + rejection: "unavailable", |
| 181 | + description: "paused", |
| 182 | + retryAfterMillis: 120_000, |
| 183 | + }), |
| 184 | + ) |
| 185 | + : Effect.succeed("done"); |
| 186 | + }); |
| 187 | + const fiber = yield* Effect.forkChild( |
| 188 | + Effect.retry(request, { schedule: relayRetrySchedule }), |
| 189 | + { |
| 190 | + startImmediately: true, |
| 191 | + }, |
| 192 | + ); |
| 193 | + yield* TestClock.adjust("90 seconds"); |
| 194 | + expect(attempts).toBe(1); |
| 195 | + yield* TestClock.adjust("31 seconds"); |
| 196 | + expect(yield* Fiber.join(fiber)).toBe("done"); |
| 197 | + expect(attempts).toBe(2); |
| 198 | + }), |
| 199 | + ); |
| 200 | +}); |
| 201 | + |
| 202 | +describe("makeManagedTunnelRecovery", () => { |
| 203 | + const config = { |
| 204 | + providerKind: "cloudflare_tunnel" as const, |
| 205 | + connectorToken: "connector-token", |
| 206 | + tunnelId: "tunnel-1", |
| 207 | + }; |
| 208 | + |
| 209 | + it.effect("retries an unavailable relay every cooldown, and cools down from when it ends", () => |
| 210 | + Effect.gen(function* () { |
| 211 | + const attemptMinutes: Array<number> = []; |
| 212 | + const recover = makeManagedTunnelRecovery({ |
| 213 | + recover: () => |
| 214 | + Effect.gen(function* () { |
| 215 | + attemptMinutes.push((yield* Clock.currentTimeMillis) / 60_000); |
| 216 | + return attemptMinutes.length < 4 |
| 217 | + ? yield* Effect.fail("relay unavailable" as const) |
| 218 | + : true; |
| 219 | + }), |
| 220 | + isRetryable: () => true, |
| 221 | + }); |
| 222 | + const run = yield* recover; |
| 223 | + const fiber = yield* Effect.forkChild(run(config), { startImmediately: true }); |
| 224 | + yield* TestClock.adjust("6 minutes"); |
| 225 | + expect(yield* Fiber.join(fiber)).toBe(true); |
| 226 | + expect(attemptMinutes.map(Math.round)).toEqual([0, 2, 4, 6]); |
| 227 | + |
| 228 | + yield* TestClock.adjust("1 minute"); |
| 229 | + const next = yield* Effect.forkChild(run(config), { startImmediately: true }); |
| 230 | + yield* TestClock.adjust("10 minutes"); |
| 231 | + yield* Fiber.join(next); |
| 232 | + expect(attemptMinutes.map(Math.round)).toEqual([0, 2, 4, 6, 10]); |
| 233 | + }), |
| 234 | + ); |
| 235 | +}); |
| 236 | + |
| 237 | +describe("retryManagedTunnelRegistration fallback", () => { |
| 238 | + it.effect("falls back by the end of the window and keeps honoring Retry-After", () => |
| 239 | + Effect.gen(function* () { |
| 240 | + let fallbackAtMillis: number | undefined; |
| 241 | + const attemptMillis: Array<number> = []; |
| 242 | + const registration = Clock.currentTimeMillis.pipe( |
| 243 | + Effect.tap((now) => Effect.sync(() => attemptMillis.push(now))), |
| 244 | + Effect.andThen( |
| 245 | + Effect.fail( |
| 246 | + new RelayRequestError({ |
| 247 | + rejection: "unavailable", |
| 248 | + description: "paused", |
| 249 | + retryAfterMillis: 900_000, |
| 250 | + }), |
| 251 | + ), |
| 252 | + ), |
| 253 | + ); |
| 254 | + yield* Effect.forkChild( |
| 255 | + retryManagedTunnelRegistration( |
| 256 | + registration, |
| 257 | + () => true, |
| 258 | + Clock.currentTimeMillis.pipe( |
| 259 | + Effect.tap((now) => |
| 260 | + Effect.sync(() => { |
| 261 | + fallbackAtMillis ??= now; |
| 262 | + }), |
| 263 | + ), |
| 264 | + Effect.asVoid, |
| 265 | + ), |
| 266 | + ), |
| 267 | + { startImmediately: true }, |
| 268 | + ); |
| 269 | + yield* TestClock.adjust("1 hour"); |
| 270 | + expect(fallbackAtMillis).toBeDefined(); |
| 271 | + expect(fallbackAtMillis).toBeLessThanOrEqual(10 * 60_000); |
| 272 | + expect(attemptMillis.length).toBeGreaterThan(2); |
| 273 | + for (let index = 1; index < attemptMillis.length; index++) { |
| 274 | + expect(attemptMillis[index]! - attemptMillis[index - 1]!).toBeGreaterThanOrEqual(900_000); |
| 275 | + } |
| 276 | + }), |
| 277 | + ); |
| 278 | +}); |
| 279 | + |
| 280 | +describe("retryDesiredLinkReconcile", () => { |
| 281 | + const relayUnavailable = new RelayRequestError({ |
| 282 | + rejection: "unavailable", |
| 283 | + description: "relay unavailable", |
| 284 | + }); |
| 285 | + |
| 286 | + it.effect("gives up on a local failure once the retry window ends", () => |
| 287 | + Effect.gen(function* () { |
| 288 | + let attempts = 0; |
| 289 | + let windowLogs = 0; |
| 290 | + const fiber = yield* Effect.forkChild( |
| 291 | + Effect.flip( |
| 292 | + retryDesiredLinkReconcile( |
| 293 | + Effect.suspend(() => { |
| 294 | + attempts += 1; |
| 295 | + return Effect.fail("cloudflared cannot run" as const); |
| 296 | + }), |
| 297 | + () => true, |
| 298 | + () => |
| 299 | + Effect.sync(() => { |
| 300 | + windowLogs += 1; |
| 301 | + }), |
| 302 | + ), |
| 303 | + ), |
| 304 | + { startImmediately: true }, |
| 305 | + ); |
| 306 | + yield* TestClock.adjust("2 hours"); |
| 307 | + expect(yield* Fiber.join(fiber)).toBe("cloudflared cannot run"); |
| 308 | + expect(attempts).toBeLessThan(20); |
| 309 | + expect(windowLogs).toBe(0); |
| 310 | + }), |
| 311 | + ); |
| 312 | + |
| 313 | + it.effect("keeps relinking while the relay is down and succeeds when it returns", () => |
| 314 | + Effect.gen(function* () { |
| 315 | + let windowLogs = 0; |
| 316 | + const reconcile = Effect.gen(function* () { |
| 317 | + return (yield* Clock.currentTimeMillis) < 30 * 60_000 |
| 318 | + ? yield* Effect.fail(relayUnavailable) |
| 319 | + : "managed"; |
| 320 | + }); |
| 321 | + const fiber = yield* Effect.forkChild( |
| 322 | + retryDesiredLinkReconcile( |
| 323 | + reconcile, |
| 324 | + () => true, |
| 325 | + () => |
| 326 | + Effect.sync(() => { |
| 327 | + windowLogs += 1; |
| 328 | + }), |
| 329 | + ), |
| 330 | + { startImmediately: true }, |
| 331 | + ); |
| 332 | + yield* TestClock.adjust("45 minutes"); |
| 333 | + expect(yield* Fiber.join(fiber)).toBe("managed"); |
| 334 | + expect(windowLogs).toBe(1); |
| 335 | + }), |
| 336 | + ); |
| 337 | +}); |
0 commit comments