Skip to content

Commit ed3b8e6

Browse files
committed
fix(daemon): probe a live daemon again before replacing it as unreachable
A 500 ms connect probe measures wall-clock time on the client's event loop. A client that stalls past it (large sync parse, GC on a loaded CI host) reads a listening daemon as unreachable, and the takeover kills it, ending every live session. The next command then meets no session: 2 devices match equally, Daemon request timed out, Invalid daemon response. While the recorded daemon process is still ours, probe up to three more times before replacing it.
1 parent 8fe03de commit ed3b8e6

2 files changed

Lines changed: 196 additions & 1 deletion

File tree

Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
import assert from 'node:assert/strict';
2+
import fs from 'node:fs';
3+
import net from 'node:net';
4+
import path from 'node:path';
5+
import { test, vi } from 'vitest';
6+
import { runCmdBackground } from '@agent-device/host-kit/command';
7+
import { computeDaemonCodeSignature } from '@agent-device/host-kit/code-signature';
8+
import {
9+
isProcessAlive,
10+
readProcessCommand,
11+
readProcessStartTime,
12+
waitForProcessExit,
13+
} from '@agent-device/host-kit/process';
14+
import { findProjectRoot, readVersion } from '@agent-device/host-kit/version';
15+
import { resolveDaemonPaths } from '../../daemon-resolution.ts';
16+
import { sendToDaemon } from '../daemon-client.ts';
17+
import {
18+
closeLoopbackServer,
19+
listenOnLoopback,
20+
supportsLoopbackBind,
21+
} from '../../__tests__/test-utils/loopback.ts';
22+
import { mkdtempForTestSync } from '../../__tests__/test-utils/tmp-dir.ts';
23+
24+
// The spawned stand-in's identity is read once and pinned: a second real `ps` can miss its
25+
// deadline under suite load and misclassify the live process as gone.
26+
const { mockReadProcessStartTime, mockReadProcessCommand } = vi.hoisted(() => ({
27+
mockReadProcessStartTime: vi.fn<(pid: number) => string | null | undefined>(),
28+
mockReadProcessCommand: vi.fn<(pid: number) => string | null | undefined>(),
29+
}));
30+
31+
vi.mock('@agent-device/host-kit/process', async (importOriginal) => {
32+
const actual = await importOriginal<typeof import('@agent-device/host-kit/process')>();
33+
return {
34+
...actual,
35+
readProcessStartTime: (pid: number) =>
36+
mockReadProcessStartTime(pid) ?? actual.readProcessStartTime(pid),
37+
readProcessCommand: (pid: number) =>
38+
mockReadProcessCommand(pid) ?? actual.readProcessCommand(pid),
39+
};
40+
});
41+
42+
function resolveCurrentDaemonCodeSignature(): string {
43+
const root = findProjectRoot();
44+
const distPath = path.join(root, 'dist', 'src', 'internal', 'daemon.js');
45+
const sourcePath = path.join(root, 'src', 'daemon.ts');
46+
const entryPath =
47+
process.execArgv.includes('--experimental-strip-types') || !fs.existsSync(distPath)
48+
? sourcePath
49+
: distPath;
50+
return computeDaemonCodeSignature(entryPath, root);
51+
}
52+
53+
/** Blocks the event loop right after the first socket arms its timeout, as a loaded client does. */
54+
function stallAfterFirstProbeArms(): { stalled: () => boolean; restore: () => void } {
55+
const originalCreateConnection = net.createConnection;
56+
let stalled = false;
57+
(net as unknown as { createConnection: typeof net.createConnection }).createConnection = ((
58+
...args: Parameters<typeof net.createConnection>
59+
) => {
60+
const socket = originalCreateConnection(...args);
61+
if (stalled) return socket;
62+
stalled = true;
63+
const armTimeout = socket.setTimeout.bind(socket);
64+
socket.setTimeout = ((timeoutMs: number) => {
65+
armTimeout(timeoutMs);
66+
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 700);
67+
return socket;
68+
}) as typeof socket.setTimeout;
69+
return socket;
70+
}) as typeof net.createConnection;
71+
return {
72+
stalled: () => stalled,
73+
restore: () => {
74+
(net as unknown as { createConnection: typeof net.createConnection }).createConnection =
75+
originalCreateConnection;
76+
},
77+
};
78+
}
79+
80+
test('sendToDaemon keeps a live socket daemon whose probe the client stalled past', async (t) => {
81+
if (!(await supportsLoopbackBind())) {
82+
t.skip('loopback listeners are not permitted in this environment');
83+
return;
84+
}
85+
86+
const stateDir = mkdtempForTestSync('agent-device-stalled-probe-');
87+
const root = mkdtempForTestSync('agent-device-stalled-probe-daemon-');
88+
const daemonDir = path.join(root, 'agent-device', 'dist', 'src', 'internal');
89+
const daemonScriptPath = path.join(daemonDir, 'daemon.js');
90+
fs.mkdirSync(daemonDir, { recursive: true });
91+
fs.writeFileSync(daemonScriptPath, 'setInterval(() => {}, 1000);\n', 'utf8');
92+
const daemonProcess = runCmdBackground(process.execPath, [daemonScriptPath], {
93+
stdio: 'ignore',
94+
allowFailure: true,
95+
captureOutput: false,
96+
});
97+
void daemonProcess.wait.catch(() => {});
98+
const pid = daemonProcess.child.pid;
99+
assert.ok(pid, 'spawned child should have a pid');
100+
const server = net.createServer((socket) => {
101+
let requestBody = '';
102+
socket.setEncoding('utf8');
103+
socket.on('data', (chunk) => {
104+
requestBody += chunk;
105+
if (!requestBody.includes('\n')) return;
106+
socket.end(`${JSON.stringify({ ok: true, data: { via: 'live-daemon' } })}\n`);
107+
});
108+
});
109+
let stall: ReturnType<typeof stallAfterFirstProbeArms> | undefined;
110+
111+
try {
112+
await new Promise((resolve) => setTimeout(resolve, 50));
113+
const processStartTime = readProcessStartTime(pid) ?? undefined;
114+
const command = readProcessCommand(pid);
115+
if (command === null || processStartTime === undefined) {
116+
t.skip('process command/start inspection is unavailable in this environment');
117+
return;
118+
}
119+
mockReadProcessStartTime.mockImplementation((queriedPid: number) =>
120+
queriedPid === pid ? processStartTime : undefined,
121+
);
122+
mockReadProcessCommand.mockImplementation((queriedPid: number) =>
123+
queriedPid === pid ? command : undefined,
124+
);
125+
const port = await listenOnLoopback(server);
126+
const paths = resolveDaemonPaths(stateDir);
127+
fs.mkdirSync(paths.baseDir, { recursive: true });
128+
fs.writeFileSync(
129+
paths.infoPath,
130+
`${JSON.stringify({
131+
port,
132+
transport: 'socket',
133+
token: 'local-secret',
134+
pid,
135+
version: readVersion(),
136+
codeSignature: resolveCurrentDaemonCodeSignature(),
137+
processStartTime,
138+
})}\n`,
139+
'utf8',
140+
);
141+
stall = stallAfterFirstProbeArms();
142+
143+
const response = await sendToDaemon({
144+
session: 'default',
145+
command: 'stalled-probe-smoke',
146+
positionals: [],
147+
flags: { stateDir, daemonTransport: 'socket' },
148+
meta: { requestId: 'req-stalled-probe' },
149+
});
150+
151+
assert.equal(stall.stalled(), true);
152+
assert.deepEqual(response, { ok: true, data: { via: 'live-daemon' } });
153+
assert.equal(isProcessAlive(pid), true);
154+
} finally {
155+
stall?.restore();
156+
mockReadProcessStartTime.mockReset();
157+
mockReadProcessCommand.mockReset();
158+
await closeLoopbackServer(server);
159+
if (isProcessAlive(pid)) {
160+
process.kill(pid, 'SIGKILL');
161+
await waitForProcessExit(pid, 1_500);
162+
}
163+
fs.rmSync(stateDir, { recursive: true, force: true });
164+
fs.rmSync(root, { recursive: true, force: true });
165+
}
166+
});

‎src/daemon-client/daemon-client-lifecycle.ts‎

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import { shellQuoteIfNeeded } from '@agent-device/kernel/device-shell';
1010
import { emitDiagnostic } from '@agent-device/host-kit/diagnostics';
1111
import { sleep } from '@agent-device/host-kit/retry';
1212

13+
import { isAgentDeviceDaemonProcess } from '../daemon-process.ts';
1314
import { findUnrecoveredRepairCommitFailure } from '../session-repair-tombstone.ts';
1415
import {
1516
resolveDaemonPaths,
@@ -72,6 +73,8 @@ type DaemonStartupWaitResult =
7273
| { kind: 'timeout' };
7374

7475
const DAEMON_STARTUP_TIMEOUT_MS = 15_000;
76+
const LIVE_DAEMON_PROBE_RETRIES = 3;
77+
const LIVE_DAEMON_PROBE_RETRY_DELAY_MS = 200;
7578
const DAEMON_STARTUP_ATTEMPTS = 2;
7679
const DAEMON_STARTUP_LOG_TAIL_BYTES = 64_000;
7780
const LOOPBACK_BLOCK_LIST = new net.BlockList();
@@ -185,7 +188,7 @@ async function readReusableLocalDaemon(settings: DaemonClientSettings): Promise<
185188
const existing = readDaemonInfo(settings.paths.infoPath);
186189
if (!existing) return null;
187190

188-
const viaClientTransport = await canConnectReusableDaemon(existing, settings.transportPreference);
191+
const viaClientTransport = await canReachReusableDaemon(existing, settings.transportPreference);
189192
const decision = await resolveDaemonTakeover(existing, {
190193
viaClientTransport,
191194
onAnyAdvertisedTransport: async () =>
@@ -202,6 +205,32 @@ async function readReusableLocalDaemon(settings: DaemonClientSettings): Promise<
202205
return null;
203206
}
204207

208+
/**
209+
* A daemon whose process is still the one its metadata recorded is probed again before it can be
210+
* judged unreachable. A probe's budget is wall-clock time on this client's event loop, so a client
211+
* that stalls past it (a large synchronous parse, a GC pause on a loaded host) reads a listening
212+
* daemon as unreachable, and replacing it ends every session the daemon holds.
213+
*/
214+
async function canReachReusableDaemon(
215+
info: DaemonInfo,
216+
preference: DaemonTransportPreference,
217+
): Promise<boolean> {
218+
if (await canConnectReusableDaemon(info, preference)) return true;
219+
for (let retry = 1; retry <= LIVE_DAEMON_PROBE_RETRIES; retry += 1) {
220+
if (!isAgentDeviceDaemonProcess(info.pid, info.processStartTime)) return false;
221+
await sleep(LIVE_DAEMON_PROBE_RETRY_DELAY_MS);
222+
if (await canConnectReusableDaemon(info, preference)) {
223+
emitDiagnostic({
224+
level: 'warn',
225+
phase: 'daemon_probe_recovered',
226+
data: { pid: info.pid, retry },
227+
});
228+
return true;
229+
}
230+
}
231+
return false;
232+
}
233+
205234
async function canConnectReusableDaemon(
206235
info: DaemonInfo,
207236
preference: DaemonTransportPreference,

0 commit comments

Comments
 (0)