Repository navigation
Expand file tree
/
Copy pathdaemon-client-timeout.ts
More file actions
208 lines (199 loc) · 10.4 KB
/
Copy pathdaemon-client-timeout.ts
File metadata and controls
208 lines (199 loc) · 10.4 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
import { AppError, normalizeError } from '@agent-device/kernel/errors';
import { emitDiagnostic } from '@agent-device/host-kit/diagnostics';
import type { OwnerIdentity } from '@agent-device/host-kit/process';
import { isAgentDeviceDaemonProcess } from '../daemon-process.ts';
import { PUBLIC_COMMANDS } from '@agent-device/command-registry/catalog';
import { resolveCommandTimeoutPolicy } from '@agent-device/command-registry/registry';
import type { RegisteredDaemonOwnership } from '../daemon-registration.ts';
import type { DaemonPaths } from '../daemon-resolution.ts';
import type { PlatformSelector } from '@agent-device/kernel/device';
import {
removeDaemonInfo,
stopDaemonProcessForTakeover,
type DaemonInfo,
} from './daemon-client-metadata.ts';
// Recovery for a timed-out request is scoped to that request or to nothing. The daemon owns
// request-scoped recovery (#3177): destroying the timed-out connection cancels exactly that
// request, and the cancel retires the runner work the request owned. The client used to recover
// host-scoped on both ends — a `pkill -f` sweep matching every agent-device runner xcodebuild,
// which sabotaged that cancel path (a swept kill reads as a host failure, not a canceled request)
// and duplicated cleanup the daemon scopes to its own leases — plus a daemon SIGKILL that ended
// every sibling session. The SIGKILL survives only behind the liveness probe below: the question
// the reset used to assume. That probe and the registration fence load on demand: a timed-out
// request is their only caller, and the CLI's eager closure (the ADR 0019 loading-shape probe)
// must not pay for a recovery path a healthy invocation never runs.
export async function handleRequestTimeout(
params: Readonly<{
info: DaemonInfo;
statePaths: DaemonPaths;
remote: boolean;
timeoutMs: number;
requestId: string | undefined;
command: string | undefined;
platform: PlatformSelector | undefined;
/** Named together so the recovery hint cannot be assembled from a swapped session and action. */
session?: string;
action?: string;
}>,
): Promise<AppError> {
const { info, statePaths, remote, timeoutMs, requestId, command, platform, session, action } =
params;
// The command's declared policy (`timeoutPolicy.onTimeout`, ADR 0008) decides whether a local
// timed-out request is RESET-ELIGIBLE; the liveness probe decides whether the eligibility runs.
// A remote client cannot reach the host's process table or the daemon's state dir, so a remote
// timeout stays purely declarative here.
const resetEligible = !remote && shouldResetDaemonAfterRequestTimeout(command);
// A daemon that answers the probe is busy or slow, not gone: the request-scoped cancel its
// transport performed when this client's connection died is the recovery that request needed,
// and sibling sessions survive. Only a daemon answering neither endpoint in the probe window is
// the hung daemon a reset was designed for.
const probeAnswered = resetEligible
? await (
await import('./daemon-client-liveness-probe.ts')
).probeDaemonResponsive(info, {
session,
})
: undefined;
const unresponsive = probeAnswered === false;
const daemonReset = unresponsive
? await resetDaemonAfterTimeout(info, statePaths)
: { performedReset: false, forcedKill: false, registration: undefined };
emitDiagnostic({
level: 'error',
phase: 'daemon_request_timeout',
data: {
timeoutMs,
requestId,
command,
daemonPidReset: daemonReset.performedReset ? info.pid : undefined,
daemonPidForceKilled: daemonReset.performedReset ? daemonReset.forcedKill : undefined,
daemonRegistrationOwnership: daemonReset.registration?.state,
daemonPreservedAfterTimeout: !remote && !daemonReset.performedReset,
daemonLivenessProbeAnswered: probeAnswered,
daemonBaseUrl: info.baseUrl,
},
});
return new AppError('COMMAND_FAILED', 'Daemon request timed out', {
timeoutMs,
requestId,
reason: 'daemon_transport_timeout',
hint: resolveRequestTimeoutHint({
remote,
resetDaemon: daemonReset.performedReset,
command,
applePlatformDeclared: isAffirmativelyApplePlatform(platform),
session,
action,
}),
});
}
// Whether a timed-out request is eligible to tear down the local daemon is declared on the
// command's descriptor (ADR 0008, `timeoutPolicy.onTimeout`): read-only capture/polling commands
// preserve the daemon so sessions survive and evidence commands still work; everything else is
// eligible. Execution of the eligibility is gated by the liveness probe — the descriptor says a
// reset is allowed where the daemon is unreachable, never that a slow-but-alive daemon may lose
// every session it owns. Unknown/undefined commands fall back to the default reset-eligible
// policy, which matches the old hand lists: not listed meant default envelope + reset.
function shouldResetDaemonAfterRequestTimeout(command: string | undefined): boolean {
return resolveCommandTimeoutPolicy(command).onTimeout === 'reset-daemon';
}
// `--platform` selectors that AFFIRMATIVELY name (or alias) an Apple device. This is the
// hint-wording gate: the hint names Apple-runner involvement only where this call site has
// evidence for it — an explicitly declared Apple platform selector. The sweep that used to offer
// "terminated something" as evidence is gone (#3177), and an undeclared or declared non-Apple
// platform is not evidence of anything.
const AFFIRMATIVE_APPLE_PLATFORM_SELECTORS: ReadonlySet<PlatformSelector> = new Set([
'apple',
'ios',
'macos',
]);
function isAffirmativelyApplePlatform(platform: PlatformSelector | undefined): boolean {
return platform !== undefined && AFFIRMATIVE_APPLE_PLATFORM_SELECTORS.has(platform);
}
// Exported for direct hint-matrix testing: handleRequestTimeout also runs the real liveness probe
// and process-kill side effects, so its wording is verified through this pure sub-function
// (see also the production-seam route tests in
// src/daemon-client/__tests__/daemon-client-timeout-route.test.ts, which prove the liveness-gated
// recovery this route performs — the side of the contract a pure formatter test cannot reach).
export function resolveRequestTimeoutHint(params: {
remote: boolean;
resetDaemon: boolean;
command: string | undefined;
/** Only ever consulted on the kept-alive branch, where an Apple hint has evidence to stand on. */
applePlatformDeclared: boolean;
/** The request's first positional, for commands whose recovery depends on which action ran. */
action?: string;
session?: string;
}): string {
const { remote, resetDaemon, command, applePlatformDeclared, session, action } = params;
if (remote) {
// A remote daemon survives this client window, so a `record stop` that ran out of time is still
// exporting there and its finished file stays retrievable by asking again.
if (command === PUBLIC_COMMANDS.record && action === 'stop') {
return `The remote daemon is still exporting the recording. Run agent-device record stop${
session ? ` --session ${session}` : ''
} again to wait for that export and receive the completed recording.`;
}
return 'Retry with --debug and verify the remote daemon URL, auth token, and remote host logs.';
}
if (resetDaemon) {
// The reset SIGKILLs the daemon pid only; child processes it owned (Apple runners included)
// are not proven stopped from here, so the hint claims nothing about them.
return 'Retry with --debug and check daemon diagnostics logs. The daemon did not answer the liveness probe and was reset after the timeout.';
}
const iosPrepareHint =
applePlatformDeclared && command === PUBLIC_COMMANDS.snapshot
? ' If this was the first Apple-platform snapshot on the device, run agent-device prepare ios-runner with the same --platform before snapshot/test so runner startup is handled explicitly.'
: '';
// The daemon canceled the request when this client's connection was destroyed, and stayed
// reachable (the probe answered, or the command's policy forbids the reset), so the session
// survives and can still be closed or inspected.
return `Retry with --debug and check daemon diagnostics logs. The timed-out ${
command ?? 'request'
} request was canceled; the daemon was kept alive so the session can still be closed or inspected.${iosPrepareHint}`;
}
type DaemonReset = Readonly<{
performedReset: boolean;
forcedKill: boolean;
/** The `daemon.json` verdict read AFTER the kill, against the identity this client started from. */
registration: RegisteredDaemonOwnership;
}>;
// The reset a liveness probe proved necessary: SIGKILL the daemon this client's metadata still
// proves is ours, then clear the registration the dead daemon can no longer release. Identity is
// re-verified immediately before the signal so a recycled pid is never signaled, and the
// registration is re-read afterward under the same proof (#3125): the probe window is exactly when
// a replacement daemon can publish, so only a `match` verdict may delete the record.
//
// The startup lock is deliberately untouched: ADR 0030 gives reclaim to the acquirer under its
// mutation guard, and a client deleting `daemon.lock` out-of-band is the legacy-reclaimer pattern
// that ADR refuses.
async function resetDaemonAfterTimeout(info: DaemonInfo, paths: DaemonPaths): Promise<DaemonReset> {
let forcedKill = false;
try {
if (isAgentDeviceDaemonProcess(info.pid, info.processStartTime)) {
process.kill(info.pid, 'SIGKILL');
forcedKill = true;
}
} catch {
void stopDaemonProcessForTakeover(info).catch((error: unknown) => {
emitDiagnostic({
level: 'warn',
phase: 'daemon_timeout_stop_failed',
data: { error: normalizeError(error) },
});
});
}
const { readRegisteredDaemonOwnership } = await import('../daemon-registration.ts');
const owner: OwnerIdentity = {
pid: info.pid,
startTime: info.processStartTime ?? null,
};
const registration = readRegisteredDaemonOwnership(paths.infoPath, owner);
// Only `match` proves the record is still the killed daemon's; every other verdict — a
// replacement's record, a pid-only record the fence refuses to trust, an unreadable one — leaves
// it for the stale-metadata paths, which re-check liveness before acting on it.
if (registration.state === 'match') {
removeDaemonInfo(paths.infoPath);
}
return { performedReset: true, forcedKill, registration };
}