Skip to content

Commit 292b569

Browse files
committed
fix(server): SSH device host no longer reconnects on a single slow health probe
The health loop closed the SSH forward and reconnected on the first missed /readyz or /health probe. A loaded host can miss one probe while still working, so reconnect only after three consecutive misses.
1 parent cefb544 commit 292b569

2 files changed

Lines changed: 81 additions & 1 deletion

File tree

‎apps/server/src/device/SshDeviceHost.test.ts‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import * as Stream from "effect/Stream";
1111
import * as HttpClient from "effect/http/HttpClient";
1212
import * as HttpClientResponse from "effect/http/HttpClientResponse";
1313
import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner";
14+
import * as TestClock from "effect/testing/TestClock";
1415
import * as ServerConfig from "../config.ts";
1516
import * as DeviceHost from "./DeviceHost.ts";
1617
import * as SshDeviceHost from "./SshDeviceHost.ts";
@@ -309,3 +310,76 @@ it.effect("lists Android devices without probing the host again once it reports
309310
yield* host.stop;
310311
}).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))),
311312
);
313+
314+
it.effect(
315+
"keeps the SSH forward through slow health probes and reconnects once the hub stays down",
316+
() =>
317+
Effect.gen(function* () {
318+
const fs = yield* FileSystem.FileSystem;
319+
const home = yield* fs.makeTempDirectoryScoped();
320+
let forwardSpawns = 0;
321+
let hubAnswers = true;
322+
const spawner = ChildProcessSpawner.make((command) =>
323+
Effect.gen(function* () {
324+
if (command._tag !== "StandardCommand") return yield* Effect.die("Unexpected command");
325+
const forwarding = command.args.includes("-L");
326+
// A fresh forward reaches a working hub again.
327+
if (forwarding && ++forwardSpawns > 1) hubAnswers = true;
328+
const output = JSON.stringify({
329+
nodePath: "/node",
330+
platforms: [{ platform: "ios", available: true }],
331+
hubPort: 1234,
332+
helpers: { serveSimAxSettings: null, serveSimCli: null },
333+
});
334+
return ChildProcessSpawner.makeHandle({
335+
pid: ChildProcessSpawner.ProcessId(123),
336+
stdout: Stream.make(new TextEncoder().encode(output)),
337+
stderr: Stream.empty,
338+
all: Stream.empty,
339+
exitCode: forwarding ? Effect.never : Effect.succeed(ChildProcessSpawner.ExitCode(0)),
340+
isRunning: Effect.succeed(forwarding),
341+
kill: () => Effect.void,
342+
stdin: Sink.drain,
343+
getInputFd: () => Sink.drain,
344+
getOutputFd: () => Stream.empty,
345+
unref: Effect.succeed(Effect.void),
346+
});
347+
}),
348+
);
349+
const host = yield* SshDeviceHost.make({
350+
kind: "ssh",
351+
id: "test",
352+
label: "Test",
353+
target: "test.example",
354+
}).pipe(
355+
Effect.provide(Layer.mergeAll(ServerConfig.layerTest(home, home), Net.layer)),
356+
Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner),
357+
Effect.provideService(
358+
HttpClient.HttpClient,
359+
HttpClient.make((request) =>
360+
hubAnswers
361+
? Effect.succeed(HttpClientResponse.fromWeb(request, new Response("ok")))
362+
: Effect.never,
363+
),
364+
),
365+
);
366+
yield* host.ensureReady(() => Effect.void);
367+
expect(forwardSpawns).toBe(1);
368+
369+
// A loaded host can miss a probe or two without being gone; the forward stays up.
370+
hubAnswers = false;
371+
yield* TestClock.adjust("30 seconds");
372+
hubAnswers = true;
373+
yield* TestClock.adjust("2 minutes");
374+
expect(forwardSpawns).toBe(1);
375+
376+
// A hub that keeps missing probes is still replaced.
377+
hubAnswers = false;
378+
yield* TestClock.adjust("1 minute");
379+
expect(forwardSpawns).toBe(2);
380+
yield* host.stop;
381+
}).pipe(
382+
Effect.scoped,
383+
Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer, TestClock.layer())),
384+
),
385+
);

‎apps/server/src/device/SshDeviceHost.ts‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@ import * as DeviceHost from "./DeviceHost.ts";
2727
import { makeDeviceHubOperations } from "./DeviceHubOperations.ts";
2828
import { quoteRemoteArg, remoteDeviceEnvironment, remoteDeviceScript } from "./sshDeviceScript.ts";
2929

30+
const UNHEALTHY_PROBE_LIMIT = 3;
31+
3032
const Probe = Schema.Struct({
3133
nodePath: Schema.String,
3234
tools: Schema.optional(DeviceToolVersions),
@@ -365,7 +367,10 @@ export const make = Effect.fn("SshDeviceHost.make")(function* (
365367
: next;
366368
yield* onStatus("ready");
367369
// Reconnect also repairs helpers that died while SSH itself stayed connected.
370+
// A loaded host can miss a probe or two while still working, and reconnecting
371+
// then drops the forward in use, so only consecutive misses count.
368372
const unhealthy = Effect.gen(function* () {
373+
let misses = 0;
369374
while (true) {
370375
yield* Effect.sleep("10 seconds");
371376
const alive = yield* http.get(`${next.hub.origin}/readyz`).pipe(
@@ -380,7 +385,8 @@ export const make = Effect.fn("SshDeviceHost.make")(function* (
380385
Effect.orElseSucceed(() => false),
381386
)
382387
: true;
383-
if (!alive || !daemonAlive) return;
388+
misses = alive && daemonAlive ? 0 : misses + 1;
389+
if (misses >= UNHEALTHY_PROBE_LIMIT) return;
384390
}
385391
});
386392
yield* Effect.gen(function* () {

0 commit comments

Comments
 (0)