Skip to content

Commit 542b7a6

Browse files
author
PR Batch Tester
committed
fix(server): preserve tunnel retry delay after an obsolete probe
1 parent 19da7b2 commit 542b7a6

4 files changed

Lines changed: 42 additions & 30 deletions

File tree

‎apps/server/src/cloud/CloudLink.lifecycle.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -197,7 +197,7 @@ describe("releaseManagedTunnelOnShutdown", () => {
197197
}),
198198
recoveryRequests: Stream.empty,
199199
requestRecovery: () => Effect.void,
200-
restartConnector: () => Effect.void,
200+
restartConnector: () => Effect.succeed(false),
201201
tunnelConnected: Stream.empty,
202202
inspectConnector: Effect.succeed(null),
203203
readinessChanged: Stream.empty,

‎apps/server/src/cloud/ConnectReachability.test.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -358,6 +358,11 @@ describe("ConnectReachability stale connector recovery", () => {
358358
if (nextConfig) {
359359
expect(yield* fixture.nextConnector).toBe(801);
360360
expect(yield* fixture.runtime.inspectConnector).not.toBeNull();
361+
yield* fixture.reachability.check;
362+
yield* TestClock.adjust(Duration.minutes(2));
363+
yield* fixture.reachability.check;
364+
expect(fixture.killed).toEqual([800, 801]);
365+
expect(yield* fixture.nextConnector).toBe(802);
361366
} else {
362367
expect(yield* fixture.runtime.inspectConnector).toBeNull();
363368
}

‎apps/server/src/cloud/ConnectReachability.ts‎

Lines changed: 29 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import * as Layer from "effect/Layer";
1515
import * as Option from "effect/Option";
1616
import * as Ref from "effect/Ref";
1717
import * as Stream from "effect/Stream";
18+
import * as SynchronizedRef from "effect/SynchronizedRef";
1819
import { HttpClient } from "effect/http";
1920
import { ChildProcessSpawner } from "effect/process";
2021

@@ -132,7 +133,9 @@ export function classifyReachability(
132133
return fail(
133134
"no_connector",
134135
"Cloudflare has no live connection from this machine for the tunnel.",
135-
restart,
136+
(connector.readyConnections ?? 0) > 0
137+
? "T3 Code retries the tunnel automatically. Wait a few minutes. If it stays offline, restart T3 Code."
138+
: restart,
136139
);
137140
}
138141
if (status !== null) {
@@ -212,7 +215,7 @@ const make = Effect.gen(function* () {
212215
Option.none<{ readonly order: number; readonly result: EnvironmentConnectReachability }>(),
213216
);
214217
const checkCounter = yield* Ref.make(0);
215-
const deadConnectorRef = yield* Ref.make<{
218+
const deadConnectorRef = yield* SynchronizedRef.make<{
216219
readonly streak: { readonly pid: number; readonly since: number } | null;
217220
readonly restartAfterMillis: number;
218221
}>({ streak: null, restartAfterMillis: Duration.toMillis(DEAD_CONNECTOR_RESTART_AFTER) });
@@ -284,30 +287,31 @@ const make = Effect.gen(function* () {
284287
(connector.readyConnections ?? 0) > 0
285288
? connector
286289
: null;
287-
const restartPid = yield* Ref.modify(deadConnectorRef, (state) => {
288-
if (result.status === "reachable") {
289-
return [
290-
null,
291-
{ streak: null, restartAfterMillis: Duration.toMillis(DEAD_CONNECTOR_RESTART_AFTER) },
292-
];
293-
}
294-
if (dead === null) return [null, { ...state, streak: null }];
295-
if (state.streak?.pid !== dead.pid) {
296-
return [null, { ...state, streak: { pid: dead.pid, since: now } }];
297-
}
298-
if (now - state.streak.since < state.restartAfterMillis) return [null, state];
299-
return [
300-
dead.pid,
301-
{
290+
yield* SynchronizedRef.updateEffect(deadConnectorRef, (state) =>
291+
Effect.gen(function* () {
292+
if (result.status === "reachable") {
293+
return {
294+
streak: null,
295+
restartAfterMillis: Duration.toMillis(DEAD_CONNECTOR_RESTART_AFTER),
296+
};
297+
}
298+
if (dead === null) return { ...state, streak: null };
299+
if (state.streak?.pid !== dead.pid) {
300+
return { ...state, streak: { pid: dead.pid, since: now } };
301+
}
302+
if (now - state.streak.since < state.restartAfterMillis) return state;
303+
const restarted = yield* runtime.restartConnector(dead.pid);
304+
return {
302305
streak: null,
303-
restartAfterMillis: Math.min(
304-
state.restartAfterMillis * 2,
305-
Duration.toMillis(DEAD_CONNECTOR_RESTART_AFTER_MAX),
306-
),
307-
},
308-
];
309-
});
310-
if (restartPid !== null) yield* runtime.restartConnector(restartPid);
306+
restartAfterMillis: restarted
307+
? Math.min(
308+
state.restartAfterMillis * 2,
309+
Duration.toMillis(DEAD_CONNECTOR_RESTART_AFTER_MAX),
310+
)
311+
: state.restartAfterMillis,
312+
};
313+
}),
314+
);
311315
});
312316

313317
const check = Effect.gen(function* () {

‎apps/server/src/cloud/ManagedEndpointRuntime.ts‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -64,8 +64,8 @@ export class CloudManagedEndpointRuntime extends Context.Service<
6464
readonly readinessChanged: Stream.Stream<void>;
6565
readonly recoveryRequests: Stream.Stream<RelayManagedEndpointRuntimeConfig>;
6666
readonly requestRecovery: (config: RelayManagedEndpointRuntimeConfig) => Effect.Effect<void>;
67-
/** Restarts the connector with this pid; ignored once it is no longer the active one. */
68-
readonly restartConnector: (pid: number) => Effect.Effect<void>;
67+
/** Returns whether the active connector matched this pid and received a restart request. */
68+
readonly restartConnector: (pid: number) => Effect.Effect<boolean>;
6969
readonly tunnelConnected: Stream.Stream<void>;
7070
readonly withLinkStateLock: <A, E, R>(effect: Effect.Effect<A, E, R>) => Effect.Effect<A, E, R>;
7171
}
@@ -727,13 +727,16 @@ export const make = Effect.gen(function* () {
727727
pid: number,
728728
) {
729729
const active = yield* Ref.get(activeRef);
730-
if (!active || Number(active.child.pid) !== pid) return;
730+
if (!active || Number(active.child.pid) !== pid) return false;
731731
yield* Effect.logWarning(
732732
"Cloudflare has no live connection from the relay client; restarting it",
733733
{ pid, tunnelId: active.config.tunnelId, tunnelName: active.config.tunnelName },
734734
);
735735
// The supervisor starts the replacement with its usual crash-loop backoff.
736-
yield* active.child.kill().pipe(Effect.ignore);
736+
return yield* active.child.kill().pipe(
737+
Effect.as(true),
738+
Effect.orElseSucceed(() => false),
739+
);
737740
}, reconcileSemaphore.withPermits(1));
738741

739742
const readMetrics = <S extends Schema.Decoder<unknown>>(url: string, schema: S) =>

0 commit comments

Comments
 (0)