Skip to content

Commit b492a9e

Browse files
voltcrashvoltcrash
committed
fix(server): remember T3 Connect HTTP/2 fallback across restarts
1 parent 15fe283 commit b492a9e

2 files changed

Lines changed: 154 additions & 5 deletions

File tree

‎apps/server/src/cloud/ManagedEndpointRuntime.test.ts‎

Lines changed: 123 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
1+
import * as NodeServices from "@effect/platform-node/NodeServices";
12
import { describe, expect, it } from "@effect/vitest";
3+
import * as Context from "effect/Context";
24
import * as Deferred from "effect/Deferred";
35
import * as Duration from "effect/Duration";
46
import * as Effect from "effect/Effect";
@@ -19,6 +21,7 @@ import type { RelayManagedEndpointRuntimeConfig } from "@t3tools/contracts/relay
1921
import * as RelayClient from "@t3tools/shared/relayClient";
2022

2123
import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
24+
import * as ServerConfig from "../config.ts";
2225
import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts";
2326

2427
const layerRelayClientAvailable = Layer.succeed(
@@ -38,29 +41,34 @@ const layerRelayClientAvailable = Layer.succeed(
3841

3942
const unusedHttpClient = HttpClient.make(() => Effect.die("unused"));
4043

44+
const emptySecretStoreLayer = Layer.mock(ServerSecretStore.ServerSecretStore)({
45+
get: () => Effect.succeedNone,
46+
set: () => Effect.void,
47+
});
48+
4149
const runtimeDependencies = (
4250
spawner: ReturnType<typeof ChildProcessSpawner.make>,
4351
relayClientLayer = layerRelayClientAvailable,
4452
httpClient = unusedHttpClient,
53+
secretStoreLayer = emptySecretStoreLayer,
4554
) =>
4655
Layer.mergeAll(
4756
Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, spawner),
4857
Layer.succeed(HttpClient.HttpClient, httpClient),
4958
relayClientLayer,
50-
Layer.mock(ServerSecretStore.ServerSecretStore)({
51-
get: () => Effect.succeedNone,
52-
}),
59+
secretStoreLayer,
5360
);
5461

5562
const buildCloudManagedEndpointRuntime = (
5663
spawner: ReturnType<typeof ChildProcessSpawner.make>,
5764
relayClientLayer = layerRelayClientAvailable,
5865
httpClient = unusedHttpClient,
66+
secretStoreLayer = emptySecretStoreLayer,
5967
) =>
6068
Effect.gen(function* () {
6169
const context = yield* Layer.build(
6270
ManagedEndpointRuntime.layer.pipe(
63-
Layer.provide(runtimeDependencies(spawner, relayClientLayer, httpClient)),
71+
Layer.provide(runtimeDependencies(spawner, relayClientLayer, httpClient, secretStoreLayer)),
6472
),
6573
);
6674
return yield* Effect.service(ManagedEndpointRuntime.CloudManagedEndpointRuntime).pipe(
@@ -403,6 +411,117 @@ describe("CloudManagedEndpointRuntime", () => {
403411
}),
404412
);
405413

414+
it.effect(
415+
"retains automatic HTTP/2 fallback across restarts until expiry and respects overrides",
416+
() =>
417+
Effect.gen(function* () {
418+
const context = yield* Layer.build(
419+
ServerSecretStore.layer.pipe(
420+
Layer.provide(ServerConfig.layerTest(process.cwd(), { prefix: "t3-transport-test-" })),
421+
Layer.provide(NodeServices.layer),
422+
),
423+
);
424+
const secretStoreLayer = Layer.succeed(
425+
ServerSecretStore.ServerSecretStore,
426+
context.pipe(Context.get(ServerSecretStore.ServerSecretStore)),
427+
);
428+
const fixture = yield* makeTransportSpawner();
429+
const start = (spawner = fixture.spawner) =>
430+
buildCloudManagedEndpointRuntime(
431+
spawner,
432+
layerRelayClientAvailable,
433+
unusedHttpClient,
434+
secretStoreLayer,
435+
);
436+
yield* Effect.gen(function* () {
437+
const runtime = yield* start();
438+
yield* runtime.applyConfig(tunnelConfig);
439+
expect(yield* Queue.take(fixture.started)).toBe(0);
440+
expect(fixture.spawned[0]!.args).not.toContain("--protocol");
441+
yield* fixture.emit(0, quicDialWarning.repeat(30));
442+
expect(yield* Queue.take(fixture.started)).toBe(1);
443+
yield* fixture.observe(runtime, 1);
444+
expect(fixture.spawned[1]!.args.slice(-3)).toEqual(["--protocol", "http2", "run"]);
445+
}).pipe(Effect.scoped);
446+
447+
yield* TestClock.adjust(Duration.hours(1));
448+
for (const transport of [undefined, "quic", "http2", "auto", undefined]) {
449+
yield* Effect.gen(function* () {
450+
const next = yield* makeTransportSpawner(transport ? { transport } : {});
451+
const restarted = yield* start(next.spawner);
452+
yield* restarted.applyConfig(tunnelConfig);
453+
if (transport === undefined) {
454+
expect(next.spawned[0]!.args.slice(-3)).toEqual(["--protocol", "http2", "run"]);
455+
} else {
456+
expect(next.spawned[0]!.args).not.toContain("--protocol");
457+
expect(next.spawned[0]!.options.env?.TUNNEL_TRANSPORT_PROTOCOL).toBe(transport);
458+
yield* next.observe(restarted, 0, quicDialWarning.repeat(30));
459+
expect(next.spawned).toHaveLength(1);
460+
}
461+
}).pipe(Effect.scoped);
462+
}
463+
464+
// Reusing the hint one hour later must not extend its original lifetime.
465+
yield* TestClock.adjust(Duration.hours(23));
466+
yield* Effect.gen(function* () {
467+
const next = yield* makeTransportSpawner();
468+
const restarted = yield* start(next.spawner);
469+
yield* restarted.applyConfig(tunnelConfig);
470+
expect(next.spawned[0]!.args).not.toContain("--protocol");
471+
expect(yield* Queue.take(next.started)).toBe(0);
472+
yield* next.emit(0, quicDatagramWarning.repeat(30));
473+
expect(yield* Queue.take(next.started)).toBe(1);
474+
expect(next.spawned[1]!.args.slice(-3)).toEqual(["--protocol", "http2", "run"]);
475+
}).pipe(Effect.scoped);
476+
}),
477+
);
478+
479+
it.effect(
480+
"ignores invalid or unreadable fallback state and still falls back if persistence fails",
481+
() =>
482+
Effect.gen(function* () {
483+
for (const cached of [
484+
"not json",
485+
'{"fellBackAt":"invalid"}',
486+
'{"fellBackAt":9007199254740991}',
487+
null,
488+
]) {
489+
yield* Effect.gen(function* () {
490+
const fixture = yield* makeTransportSpawner();
491+
const runtime = yield* buildCloudManagedEndpointRuntime(
492+
fixture.spawner,
493+
layerRelayClientAvailable,
494+
unusedHttpClient,
495+
Layer.mock(ServerSecretStore.ServerSecretStore)({
496+
get: () =>
497+
cached === null
498+
? Effect.fail(
499+
new ServerSecretStore.SecretStoreReadError({
500+
resource: "test",
501+
cause: "unreadable",
502+
}),
503+
)
504+
: Effect.succeedSome(new TextEncoder().encode(cached)),
505+
set: () =>
506+
Effect.fail(
507+
new ServerSecretStore.SecretStorePersistError({
508+
resource: "test",
509+
cause: "read-only",
510+
}),
511+
),
512+
}),
513+
);
514+
yield* runtime.applyConfig(tunnelConfig);
515+
expect(yield* Queue.take(fixture.started)).toBe(0);
516+
expect(fixture.spawned[0]!.args).not.toContain("--protocol");
517+
yield* fixture.emit(0, quicDialWarning.repeat(30));
518+
expect(yield* Queue.take(fixture.started)).toBe(1);
519+
expect(fixture.spawned[1]!.args.slice(-3)).toEqual(["--protocol", "http2", "run"]);
520+
}).pipe(Effect.scoped);
521+
}
522+
}),
523+
);
524+
406525
it.effect("requests recovery after a failed HTTP/2 spawn and retries with HTTP/2", () =>
407526
Effect.gen(function* () {
408527
const fixture = yield* makeTransportSpawner({ failSpawn: 1 });

‎apps/server/src/cloud/ManagedEndpointRuntime.ts‎

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@ import { HttpClient, HttpClientResponse } from "effect/http";
1919
import * as ChildProcess from "effect/process/ChildProcess";
2020
import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner";
2121

22+
import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
23+
2224
export type CloudManagedEndpointRuntimeStatus =
2325
| {
2426
readonly status: "disabled";
@@ -109,6 +111,12 @@ const ConnectorConfigBody = Schema.Struct({
109111
// immediately. Keep that evidence across registrations and connector restarts.
110112
const QUIC_STALLS_BEFORE_HTTP2 = 30;
111113
const QUIC_STALL_WINDOW_MS = 5 * 60_000;
114+
// Retry QUIC on a later startup after transient failures or network changes; reuse never renews this hint.
115+
const HTTP2_FALLBACK_TTL_MS = 24 * 60 * 60_000;
116+
const HTTP2_FALLBACK_STATE = "cloud-endpoint-http2-fallback";
117+
const Http2Fallback = Schema.fromJsonString(Schema.Struct({ fellBackAt: Schema.Number }));
118+
const decodeHttp2Fallback = Schema.decodeUnknownOption(Http2Fallback);
119+
const encodeHttp2Fallback = Schema.encodeSync(Http2Fallback);
112120
// A failed background install of the pinned relay client is retried no sooner
113121
// than this, so a crash-looping connector or an offline host does not redownload
114122
// on every reconcile.
@@ -182,6 +190,7 @@ const stopConnector = (connector: ActiveConnector | null) =>
182190
export const make = Effect.gen(function* () {
183191
const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
184192
const relayClient = yield* RelayClient.RelayClient;
193+
const secrets = yield* ServerSecretStore.ServerSecretStore;
185194
const httpClient = HttpClient.withScope(yield* HttpClient.HttpClient);
186195
const activeRef = yield* Ref.make<ActiveConnector | null>(null);
187196
const desiredConfigRef = yield* Ref.make<RelayManagedEndpointRuntimeConfig | null>(null);
@@ -199,7 +208,18 @@ export const make = Effect.gen(function* () {
199208
const notReadyChecksBeforeRestartRef = yield* Ref.make(CONNECTOR_NOT_READY_CHECKS_BEFORE_RESTART);
200209
// An explicit cloudflared transport in the environment is the user's choice.
201210
const transportChosenByUser = process.env.TUNNEL_TRANSPORT_PROTOCOL !== undefined;
202-
const http2Ref = yield* Ref.make(false);
211+
const previousFallback = transportChosenByUser
212+
? Option.none()
213+
: yield* secrets.get(HTTP2_FALLBACK_STATE).pipe(
214+
Effect.map(Option.flatMap((bytes) => decodeHttp2Fallback(new TextDecoder().decode(bytes)))),
215+
Effect.orElseSucceed(() => Option.none()),
216+
);
217+
const now = yield* Clock.currentTimeMillis;
218+
const http2Ref = yield* Ref.make(
219+
Option.isSome(previousFallback) &&
220+
now >= previousFallback.value.fellBackAt &&
221+
now - previousFallback.value.fellBackAt < HTTP2_FALLBACK_TTL_MS,
222+
);
203223
const quicStallsRef = yield* Ref.make<ReadonlyArray<number>>([]);
204224
let reconcileConfig: CloudManagedEndpointRuntime["Service"]["applyConfig"];
205225

@@ -452,6 +472,16 @@ export const make = Effect.gen(function* () {
452472
const desiredConfig = yield* Ref.get(desiredConfigRef);
453473
if ((yield* Ref.get(http2Ref)) || active !== connector || !desiredConfig) return;
454474
yield* Ref.set(http2Ref, true);
475+
yield* secrets
476+
.set(
477+
HTTP2_FALLBACK_STATE,
478+
new TextEncoder().encode(
479+
encodeHttp2Fallback({ fellBackAt: yield* Clock.currentTimeMillis }),
480+
),
481+
)
482+
.pipe(
483+
Effect.catch(() => Effect.logWarning("Could not persist the relay client HTTP/2 fallback")),
484+
);
455485
yield* Effect.logWarning("Relay client keeps losing QUIC connections; switching to HTTP/2", {
456486
pid: Number(connector.child.pid),
457487
tunnelId: connector.config.tunnelId,

0 commit comments

Comments
 (0)