Skip to content

Commit 0809263

Browse files
committed
fix(daemon-client): classify a failed restart health probe by its own timeout
1 parent 80fbf54 commit 0809263

2 files changed

Lines changed: 98 additions & 44 deletions

File tree

‎src/daemon-client/__tests__/daemon-client-transport.test.ts‎

Lines changed: 32 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -237,20 +237,51 @@ test('a restart health probe that times out just before the RPC deadline still r
237237
});
238238
const clock = vi
239239
.spyOn(performance, 'now')
240-
.mockImplementation(() => realNow() - (probing ? 2 : 0));
240+
.mockImplementation(() => realNow() - (probing ? 50 : 0));
241241
try {
242242
const port = await listenOnLoopback(server);
243243
await assert.rejects(
244244
sendWithStaleInstance(port, 150),
245245
(error: unknown) =>
246246
error instanceof AppError && error.details?.reason === 'daemon_transport_timeout',
247247
);
248+
assert.equal(probing, true);
248249
} finally {
249250
clock.mockRestore();
250251
await closeLoopbackServer(server);
251252
}
252253
});
253254

255+
test('a restart health probe refused near the RPC deadline reports the daemon as unavailable', async (t) => {
256+
if (await skipWhenLoopbackUnavailable(t)) return;
257+
let probed = false;
258+
const server = http.createServer((req, res) => {
259+
if (req.url === '/health') {
260+
probed = true;
261+
res.statusCode = 503;
262+
res.end();
263+
return;
264+
}
265+
res.statusCode = 409;
266+
res.setHeader(DAEMON_HTTP_INSTANCE_MISMATCH_HEADER, 'true');
267+
res.end();
268+
});
269+
try {
270+
const port = await listenOnLoopback(server);
271+
await assert.rejects(
272+
sendWithStaleInstance(port, 150),
273+
(error: unknown) =>
274+
error instanceof AppError &&
275+
error.message === 'Remote daemon is unavailable' &&
276+
error.details?.reason !== 'daemon_transport_timeout' &&
277+
error.details?.daemonBaseUrl === `http://127.0.0.1:${port}`,
278+
);
279+
assert.equal(probed, true);
280+
} finally {
281+
await closeLoopbackServer(server);
282+
}
283+
});
284+
254285
test('proxy forwards cached upstream identity and rejects a restarted upstream before dispatch', async (t) => {
255286
if (await skipWhenLoopbackUnavailable(t)) return;
256287
let upstreamInstance = 'upstream-one';

‎src/daemon-client/daemon-client-transport.ts‎

Lines changed: 66 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import type { RequestProgressSink } from '@agent-device/contracts/progress';
2+
import type http from 'node:http';
23
import net from 'node:net';
34
import { AppError } from '@agent-device/kernel/errors';
45
import { loadNodeHttpRequester, readNodeHttpResponseBody } from '@agent-device/host-kit/transport';
@@ -94,11 +95,20 @@ function canConnectHttp(info: DaemonInfo): Promise<boolean> {
9495
return readDaemonHttpHealth(info).then((health) => health.reachable);
9596
}
9697

98+
type HealthProbeResult = {
99+
health: RemoteDaemonHealth;
100+
/** The probe's own budget timer ended it, not a refusal, reset, abort or 5xx. */
101+
timedOut: boolean;
102+
};
103+
97104
export async function readRemoteDaemonHealth(
98105
info: DaemonInfo,
99106
probeTimeoutMs?: number,
107+
onProbeTimedOut?: () => void,
100108
): Promise<RemoteDaemonHealth> {
101-
const health = await readDaemonHttpHealth(info, probeTimeoutMs);
109+
const probe = await probeDaemonHttpHealth(info, probeTimeoutMs);
110+
const health = probe.health;
111+
if (probe.timedOut) onProbeTimedOut?.();
102112
if (!info.baseUrl || !health.reachable) return health;
103113
// Every link a command RPC crosses has to speak the client's protocol: a proxy that reports a
104114
// skewed daemon behind it fails here, before the RPC, exactly like a skewed proxy does.
@@ -125,21 +135,30 @@ async function readDaemonHttpHealth(
125135
info: DaemonInfo,
126136
probeTimeoutMs?: number,
127137
): Promise<RemoteDaemonHealth> {
138+
return (await probeDaemonHttpHealth(info, probeTimeoutMs)).health;
139+
}
140+
141+
async function probeDaemonHttpHealth(
142+
info: DaemonInfo,
143+
probeTimeoutMs?: number,
144+
): Promise<HealthProbeResult> {
145+
const unreachable: HealthProbeResult = { health: { reachable: false }, timedOut: false };
128146
const endpoint = info.baseUrl
129147
? buildDaemonHttpUrl(info.baseUrl, 'health')
130148
: info.httpPort
131149
? `http://127.0.0.1:${info.httpPort}/health`
132150
: null;
133-
if (!endpoint) return { reachable: false };
151+
if (!endpoint) return unreachable;
134152
const url = new URL(endpoint);
135153
const transport = await loadNodeHttpRequester(url.protocol);
136154
const timeoutMs = Math.min(
137155
info.baseUrl ? REMOTE_DAEMON_HEALTHCHECK_TIMEOUT_MS : LOCAL_DAEMON_HEALTHCHECK_TIMEOUT_MS,
138156
probeTimeoutMs ?? Number.POSITIVE_INFINITY,
139157
);
140-
if (timeoutMs <= 0) return { reachable: false };
158+
if (timeoutMs <= 0) return unreachable;
141159
return await new Promise((resolve) => {
142160
const headers = info.baseUrl ? buildDaemonHttpAuthHeaders(info.token) : {};
161+
const signal = AbortSignal.timeout(Math.ceil(timeoutMs));
143162
const req = transport.request(
144163
{
145164
protocol: url.protocol,
@@ -148,38 +167,44 @@ async function readDaemonHttpHealth(
148167
path: url.pathname + url.search,
149168
method: 'GET',
150169
timeout: timeoutMs,
151-
signal: AbortSignal.timeout(Math.ceil(timeoutMs)),
170+
signal,
152171
headers,
153172
},
154-
(res) => {
155-
let body = '';
156-
res.setEncoding('utf8');
157-
res.on('data', (chunk) => {
158-
body += chunk;
159-
});
160-
res.on('end', () => {
161-
const statusCode = res.statusCode ?? 500;
162-
resolve({
163-
reachable: statusCode < 500,
164-
statusCode,
165-
...readHealthPayload(body),
166-
});
167-
});
168-
res.on('error', () => resolve({ reachable: false }));
169-
res.on('aborted', () => resolve({ reachable: false }));
170-
},
173+
(res) => collectHealthResponse(res, signal, resolve),
171174
);
172175
req.on('timeout', () => {
173176
req.destroy();
174-
resolve({ reachable: false });
177+
resolve({ ...unreachable, timedOut: true });
175178
});
176179
req.on('error', () => {
177-
resolve({ reachable: false });
180+
resolve({ ...unreachable, timedOut: signal.aborted });
178181
});
179182
req.end();
180183
});
181184
}
182185

186+
function collectHealthResponse(
187+
res: http.IncomingMessage,
188+
signal: AbortSignal,
189+
resolve: (result: HealthProbeResult) => void,
190+
): void {
191+
const unreachable: HealthProbeResult = { health: { reachable: false }, timedOut: false };
192+
let body = '';
193+
res.setEncoding('utf8');
194+
res.on('data', (chunk) => {
195+
body += chunk;
196+
});
197+
res.on('end', () => {
198+
const statusCode = res.statusCode ?? 500;
199+
resolve({
200+
health: { reachable: statusCode < 500, statusCode, ...readHealthPayload(body) },
201+
timedOut: false,
202+
});
203+
});
204+
res.on('error', () => resolve({ ...unreachable, timedOut: signal.aborted }));
205+
res.on('aborted', () => resolve({ ...unreachable, timedOut: signal.aborted }));
206+
}
207+
183208
function readHealthPayload(body: string): Omit<RemoteDaemonHealth, 'reachable' | 'statusCode'> {
184209
try {
185210
const parsed = JSON.parse(body) as { upstream?: unknown };
@@ -254,14 +279,12 @@ async function retryAfterRemoteInstanceMismatch(
254279
timeoutMs,
255280
deadline,
256281
);
257-
const probeStartedAt = performance.now();
258-
const health = await readRemoteDaemonHealth(info, probeTimeoutMs);
259-
if (!health.reachable && probeRanToCallerDeadline(probeStartedAt, probeTimeoutMs)) {
260-
throw handleRequestTimeout({
261-
info,
262-
statePaths,
263-
...timeoutRequestContext(req, true, timeoutMs ?? 0),
264-
});
282+
let probeTimedOut = false;
283+
const health = await readRemoteDaemonHealth(info, probeTimeoutMs, () => {
284+
probeTimedOut = true;
285+
});
286+
if (probeTimedOut && timeoutMs !== undefined && isCallerDeadlineProbeBudget(probeTimeoutMs)) {
287+
throw requestTimeoutError(info, req, statePaths, timeoutMs);
265288
}
266289
const remainingMs = remainingRemoteRequestTimeoutMs(info, req, statePaths, timeoutMs, deadline);
267290
if (!health.reachable) {
@@ -280,17 +303,8 @@ async function retryAfterRemoteInstanceMismatch(
280303
}
281304
}
282305

283-
// Node timers may fire a few ms before performance.now() reaches the same instant.
284-
const PROBE_TIMER_SLOP_MS = 5;
285-
286-
function probeRanToCallerDeadline(
287-
probeStartedAt: number,
288-
probeTimeoutMs: number | undefined,
289-
): boolean {
290-
if (probeTimeoutMs === undefined || probeTimeoutMs > REMOTE_DAEMON_HEALTHCHECK_TIMEOUT_MS) {
291-
return false;
292-
}
293-
return performance.now() - probeStartedAt >= probeTimeoutMs - PROBE_TIMER_SLOP_MS;
306+
function isCallerDeadlineProbeBudget(probeTimeoutMs: number | undefined): boolean {
307+
return probeTimeoutMs !== undefined && probeTimeoutMs <= REMOTE_DAEMON_HEALTHCHECK_TIMEOUT_MS;
294308
}
295309

296310
function remainingRemoteRequestTimeoutMs(
@@ -303,7 +317,16 @@ function remainingRemoteRequestTimeoutMs(
303317
if (deadline === undefined || timeoutMs === undefined) return undefined;
304318
const remainingMs = deadline - performance.now();
305319
if (remainingMs > 0) return remainingMs;
306-
throw handleRequestTimeout({
320+
throw requestTimeoutError(info, req, statePaths, timeoutMs);
321+
}
322+
323+
function requestTimeoutError(
324+
info: DaemonInfo,
325+
req: DaemonRequest,
326+
statePaths: DaemonPaths,
327+
timeoutMs: number,
328+
): AppError {
329+
return handleRequestTimeout({
307330
info,
308331
statePaths,
309332
...timeoutRequestContext(req, true, timeoutMs),

0 commit comments

Comments
 (0)