Repository navigation
Expand file tree
/
Copy pathdaemon-client-timeout.ts
More file actions
189 lines (181 loc) · 9.03 KB
/
Copy pathdaemon-client-timeout.ts
File metadata and controls
189 lines (181 loc) · 9.03 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
import { AppError } from '@agent-device/kernel/errors';
import { runCmdSync } from '@agent-device/host-kit/command';
import { emitDiagnostic } from '@agent-device/host-kit/diagnostics';
import type { DaemonRetirementResult } from '../daemon-registration-owner.ts';
import { PUBLIC_COMMANDS } from '@agent-device/command-registry/catalog';
import { resolveCommandTimeoutPolicy } from '@agent-device/command-registry/registry';
import type { DaemonPaths } from '../daemon-resolution.ts';
import type { PlatformSelector } from '@agent-device/kernel/device';
import type { DaemonInfo } from './daemon-client-metadata.ts';
const IOS_RUNNER_XCODEBUILD_KILL_PATTERNS = [
'xcodebuild .*AgentDeviceRunnerUITests/RunnerTests/testCommand',
// A client in the field already pkills these exact bytes. This sweep ships separately from the
// runner and cannot know which version wrote a timed-out launch, so it never follows a rename:
// it must keep matching the names older writers used. The literal stays pinned here rather than
// derived from `runner-artifact-env.ts`, which builds only today's session name.
String.raw`xcodebuild .*AgentDeviceRunner\.env\.session-`,
String.raw`xcodebuild build-for-testing .*apple/runner/AgentDeviceRunner/AgentDeviceRunner\.xcodeproj`,
];
// `--platform` selectors that AFFIRMATIVELY name (or alias) an Apple device.
// This is deliberately narrower than "not proven non-Apple": the client's
// declared platform is not authoritative for session-bound execution (see
// the eligibility note on `handleRequestTimeout` below), so an undeclared or
// declared-non-Apple platform is not evidence of anything — it only counts
// as Apple evidence when it says so outright.
const AFFIRMATIVE_APPLE_PLATFORM_SELECTORS: ReadonlySet<PlatformSelector> = new Set([
'apple',
'ios',
'macos',
]);
function isAffirmativelyApplePlatform(platform: PlatformSelector | undefined): boolean {
return platform !== undefined && AFFIRMATIVE_APPLE_PLATFORM_SELECTORS.has(platform);
}
export async function handleRequestTimeout(
params: Readonly<{
info: DaemonInfo;
statePaths: DaemonPaths;
remote: boolean;
timeoutMs: number;
requestId: string | undefined;
command: string | undefined;
platform: PlatformSelector | undefined;
/** Named together so the recovery hint cannot be assembled from a swapped session and action. */
session?: string;
action?: string;
}>,
): Promise<AppError> {
const { info, statePaths, remote, timeoutMs, requestId, command, platform, session, action } =
params;
// Cleanup eligibility stays UNCONDITIONAL for every local (non-remote)
// timeout, on purpose: the request's declared --platform is not
// authoritative for session-bound execution. An existing session's real
// device platform can silently override a conflicting declared selector
// (`applyStripLockPolicy` in request-lock-policy.ts, reached via
// --session-lock strip), and the common session-bound request omits
// --platform entirely — so there is no client-visible signal that proves
// a request cannot touch an Apple runner. The pkill patterns are
// Apple-process-name-specific, so sweeping them on a non-Apple host or
// session matches nothing and costs a few no-op subprocess spawns, never
// a wrong skip.
const cleanup = remote ? { terminated: 0 } : cleanupTimedOutIosRunnerBuilds();
const resetDaemon = !remote && shouldResetDaemonAfterRequestTimeout(command);
let retirement: DaemonRetirementResult | undefined;
if (resetDaemon) {
const { stopAndRetireDaemon } = await import('../daemon-registration-owner.ts');
retirement = await stopAndRetireDaemon({
paths: statePaths,
observed: { pid: info.pid, startTime: info.processStartTime ?? null },
mode: 'force',
});
}
const forcedKill =
retirement?.status !== 'absent' &&
retirement?.termination?.status === 'exited' &&
retirement.termination.mode === 'forced';
// The HINT, unlike cleanup, may only name Apple-runner involvement on
// evidence this call site actually has: an explicitly declared Apple
// platform selector, or the cleanup itself having terminated a matching
// process (proof positive regardless of what --platform claimed). Any
// other combination — undeclared platform, declared non-Apple platform,
// zero processes terminated — gets platform-neutral wording instead of
// asserting Apple specifics the client cannot back up.
const appleCleanupEvidence = isAffirmativelyApplePlatform(platform) || cleanup.terminated > 0;
emitDiagnostic({
level: 'error',
phase: 'daemon_request_timeout',
data: {
timeoutMs,
requestId,
command,
timedOutRunnerPidsTerminated: cleanup.terminated,
timedOutRunnerCleanupError: cleanup.error,
daemonPidReset: retirement?.status === 'retired' ? info.pid : undefined,
daemonPidForceKilled: resetDaemon ? forcedKill : undefined,
daemonRetirement: retirement,
daemonPreservedAfterTimeout: !remote && !resetDaemon,
daemonBaseUrl: info.baseUrl,
},
});
return new AppError('COMMAND_FAILED', 'Daemon request timed out', {
timeoutMs,
requestId,
reason: 'daemon_transport_timeout',
...(retirement ? { retirement, stateDir: statePaths.baseDir } : {}),
hint:
retirement?.status === 'retained'
? `The daemon could not be safely retired. State was retained at ${statePaths.baseDir}. ${retirement.error?.hint ?? 'Retry with --debug and inspect daemon diagnostics before retrying.'}`
: resolveRequestTimeoutHint({
remote,
resetDaemon,
command,
appleCleanupEvidence,
session,
action,
}),
});
}
// Whether a timed-out request tears down the local daemon is declared on the
// command's descriptor (ADR 0008, `timeoutPolicy.onTimeout`): read-only
// capture/polling commands preserve the daemon so sessions survive and evidence
// commands still work; everything else resets it. Unknown/undefined commands
// fall back to the default reset-daemon policy.
function shouldResetDaemonAfterRequestTimeout(command: string | undefined): boolean {
return resolveCommandTimeoutPolicy(command).onTimeout === 'reset-daemon';
}
// Exported for direct hint-matrix testing: handleRequestTimeout also triggers
// real pkill/process-kill side effects, so its wording is verified through
// this pure sub-function rather than the full timeout path (see also the
// production-seam route tests in
// src/daemon-client/__tests__/daemon-client-timeout-route.test.ts, which
// prove the cleanup-eligibility side of this contract that a pure formatter
// test cannot).
export function resolveRequestTimeoutHint(params: {
remote: boolean;
resetDaemon: boolean;
command: string | undefined;
appleCleanupEvidence: boolean;
/** The request's first positional, for commands whose recovery depends on which action ran. */
action?: string;
session?: string;
}): string {
const { remote, resetDaemon, command, appleCleanupEvidence, session, action } = params;
if (remote) {
// A remote daemon survives this client window, so a `record stop` that ran out of time is still
// exporting there and its finished file stays retrievable by asking again. A local timeout
// resets the daemon mid-export, where that promise would be false.
if (command === PUBLIC_COMMANDS.record && action === 'stop') {
return `The remote daemon is still exporting the recording. Run agent-device record stop${
session ? ` --session ${session}` : ''
} again to wait for that export and receive the completed recording.`;
}
return 'Retry with --debug and verify the remote daemon URL, auth token, and remote host logs.';
}
if (!resetDaemon) {
const iosPrepareHint =
appleCleanupEvidence && command === PUBLIC_COMMANDS.snapshot
? ' If this was the first Apple-platform snapshot on the device, run agent-device prepare ios-runner with the same --platform before snapshot/test so runner startup is handled explicitly.'
: '';
const appleCleanupNote = appleCleanupEvidence
? ' and Apple runner work was aborted when detected'
: '';
return `Retry with --debug and check daemon diagnostics logs. The timed-out ${command ?? 'request'} request was canceled${appleCleanupNote}; the daemon was kept alive so the session can still be closed or inspected.${iosPrepareHint}`;
}
return appleCleanupEvidence
? 'Retry with --debug and check daemon diagnostics logs. Timed-out Apple runner xcodebuild processes were terminated when detected.'
: 'Retry with --debug and check daemon diagnostics logs. The daemon was reset after the timeout.';
}
function cleanupTimedOutIosRunnerBuilds(): { terminated: number; error?: string } {
let terminated = 0;
try {
for (const pattern of IOS_RUNNER_XCODEBUILD_KILL_PATTERNS) {
const result = runCmdSync('pkill', ['-f', pattern], { allowFailure: true });
if (result.exitCode === 0) terminated += 1;
}
return { terminated };
} catch (error) {
return {
terminated,
error: error instanceof Error ? error.message : String(error),
};
}
}