Repository navigation
Expand file tree
/
Copy pathdaemon-client-stop.ts
More file actions
251 lines (239 loc) · 11.4 KB
/
Copy pathdaemon-client-stop.ts
File metadata and controls
251 lines (239 loc) · 11.4 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
import {
daemonShutdownCompletionAnswers,
daemonShutdownCompletionStillRetained,
readDaemonShutdownCompletion,
type DaemonShutdownCompletion,
type DaemonShutdownCompletionQuery,
} from '@agent-device/daemon-contracts/daemon-shutdown-report';
import { waitForDaemonExit } from '@agent-device/daemon-contracts/daemon-process';
import {
DAEMON_LIFECYCLE_REASONS,
readDaemonLifecycleCensus,
type DaemonPaths,
} from '@agent-device/daemon-contracts/daemon-resolution';
import type { DaemonStopIdentity } from '@agent-device/daemon-contracts/daemon-registration';
import {
daemonNotRunningStopResult,
stopDaemon,
type DaemonStopResult,
} from '@agent-device/daemon-contracts/daemon-stop';
import { AppError } from '@agent-device/kernel/errors';
import { sleep } from '@agent-device/host-kit/retry';
import { readDaemonRecordAddress, type DaemonInfo } from './daemon-client-metadata.ts';
import {
DAEMON_LEAVE_WAIT_MS,
requestDaemonLifecycle,
type DaemonLifecycleTarget,
} from './daemon-client-lifecycle-request.ts';
// How long a stop keeps asking while the runtime refuses because it is still starting up and has no
// drain path to run yet.
const EMBEDDED_STOP_ASK_BUDGET_MS = 5_000;
const EMBEDDED_STOP_ASK_GAP_MS = 250;
const EMBEDDED_STOP_POLL_MS = 100;
// How often the wait asks whether the runtime's host is still alive. A lifetime read costs a process
// table lookup, and the answer changes on a much slower clock than the record does.
const EMBEDDED_STOP_LIFETIME_POLL_MS = 1_000;
/**
* Stops the daemon a state directory registers, by the owner its record states.
*
* The caller hands over the record it read so routing, retirement, and a `--clean` walk judge one
* observation of a file another process may be rewriting. Only a detached daemon is signaled: a host owns
* an embedded runtime and ends it with its own close, so this reaches it over its transport with a
* shutdown request. A record this build cannot place is asked the same way and the runtime answers with
* the rule it enforces. A record that cannot be read gets neither: it may describe a runtime that is
* serving, so this refuses rather than reporting an absence.
*/
export async function stopRecordedDaemon(
record: DaemonStopIdentity,
target: DaemonLifecycleTarget,
): Promise<DaemonStopResult> {
if (record.ownership === 'unreadable') throw daemonRecordUnreadableError(target.paths);
if (record.ownership === 'embedded' || record.ownership === 'unknown') {
return await stopEmbeddedDaemon(target);
}
const observed = record.identity ? { ...record.identity, ownership: 'detached' as const } : null;
return await stopDaemon({ paths: target.paths, observed });
}
/**
* A record that is present and cannot be read: not an absence, and nothing to address. A stop signals
* nothing and cannot ask a runtime it cannot reach, so it names the record instead of guessing that the
* daemon already left.
*/
function daemonRecordUnreadableError(paths: DaemonPaths): AppError {
return new AppError(
'COMMAND_FAILED',
`The daemon record on ${paths.baseDir} cannot be read, so this stop cannot tell whether a daemon runs.`,
{
reason: 'daemon-record-unreadable',
stateDir: paths.baseDir,
hint: 'Nothing was signaled. Inspect or remove that record, then retry.',
},
);
}
/**
* Stops the runtime a host process owns, over the transport its record publishes.
*
* Nothing here signals a process. A host owns this runtime, so the only stop available from outside is
* the authenticated shutdown request the runtime answers itself, followed by the bounded completion
* record it leaves when its close has a report to give.
*/
async function stopEmbeddedDaemon(target: DaemonLifecycleTarget): Promise<DaemonStopResult> {
const deadlineMs = Date.now() + DAEMON_LEAVE_WAIT_MS;
// Asking needs a transport and a token, which a record can state an owner and still omit. Such a
// record is not an absence: something holds this state directory and this stop cannot address it,
// which is a refusal rather than nothing running.
const address = readDaemonRecordAddress(target.paths.infoPath);
if (address.status === 'absent') return daemonNotRunningStopResult();
// One reason for one fact: a record that cannot be read says so the same way on every stop route.
if (address.status === 'unreadable') throw daemonRecordUnreadableError(target.paths);
if (address.status !== 'addressable') {
throw daemonEmbeddedRecordUnaddressableError(target.paths);
}
const info = address.info;
// An answer of `gone` is waited on the same way as an agreement: a daemon that stopped answering has
// either left, is leaving, or was replaced, and the two reads below are what tells those apart.
await askToLeave(info, target, Date.now() + EMBEDDED_STOP_ASK_BUDGET_MS);
const waited = await waitForClose(target.paths, instanceOf(info), deadlineMs);
if (waited === 'instance-gone') return daemonNotRunningStopResult();
if (!waited) throw daemonShutdownUnconfirmedError(info, target.paths);
return stopResultFromCompletion(waited);
}
/** The instance a stop addresses: the record's own lifetime, which is what a record is matched on. */
function instanceOf(info: { pid: number; processStartTime?: string }) {
return { pid: info.pid, processStartTime: info.processStartTime ?? '' };
}
/**
* Sends the shutdown request, asking again while the runtime refuses because it is starting up. What it
* answers is not this function's business: an agreement, a refusal that the daemon is already leaving,
* and a connection that cannot be made all send the stop to the same wait for evidence.
*/
async function askToLeave(
info: Parameters<typeof requestDaemonLifecycle>[0],
target: DaemonLifecycleTarget,
askUntilMs: number,
): Promise<void> {
for (;;) {
try {
await requestDaemonLifecycle(info, target, 'shutdown');
return;
} catch (error) {
// A runtime that is still starting has no drain path and says so with its census. That is the
// one refusal worth asking again about: it will be able to drain in a moment.
if (!refusedWhileStartingUp(error) || Date.now() >= askUntilMs) throw error;
}
await sleep(EMBEDDED_STOP_ASK_GAP_MS);
}
}
function refusedWhileStartingUp(error: unknown): boolean {
return (
error instanceof AppError &&
error.details?.reason === DAEMON_LIFECYCLE_REASONS.busy &&
readDaemonLifecycleCensus(error.details)?.startingUp === true
);
}
/**
* Reads the record the closing runtime publishes, and notices when its writer can no longer publish one:
* the instance's own lifetime is provably over, so what is on disk is the whole truth and nothing is left
* to wait for.
*
* Both reads are needed, and each answers what the other cannot. The record answers by instance rather
* than by request, so a second stop of the same runtime reads the same answer instead of sitting out its
* budget for a record written for somebody else's request. The lifetime check answers the two cases the
* record cannot: a host that died before writing its report would otherwise leave a stop waiting on a
* writer that is gone, and a record of a retained registration is evidence only while some process is
* left holding it.
*/
async function waitForClose(
paths: DaemonPaths,
instance: DaemonShutdownCompletionQuery,
deadlineMs: number,
): Promise<DaemonShutdownCompletion | 'instance-gone' | null> {
let nextLifetimeCheckMs = 0;
// Whether the instance is still the process these records name. It starts unknown, and the only record
// that can be read before the first probe is one that answers on its own, so nothing is trusted on an
// assumption.
let instanceIsLive = false;
while (Date.now() < deadlineMs) {
const completion = readDaemonShutdownCompletion(paths.baseDir);
// A record of a close that just happened is the writer's own word and settles this stop at once,
// whatever else is true about the instance.
if (completion && daemonShutdownCompletionAnswers(completion, instance)) return completion;
if (Date.now() >= nextLifetimeCheckMs) {
nextLifetimeCheckMs = Date.now() + EMBEDDED_STOP_LIFETIME_POLL_MS;
if (await instanceEnded(instance)) return 'instance-gone';
instanceIsLive = true;
}
// A record of a registration this instance still holds answers too, and the probe above is what makes
// it evidence rather than a dead host's leftovers.
if (
completion &&
instanceIsLive &&
daemonShutdownCompletionStillRetained(completion, instance)
) {
return completion;
}
await sleep(Math.min(EMBEDDED_STOP_POLL_MS, Math.max(0, deadlineMs - Date.now())));
}
return null;
}
async function instanceEnded(instance: DaemonShutdownCompletionQuery): Promise<boolean> {
if (!instance.processStartTime.trim()) return false;
const identity = { pid: instance.pid, startTime: instance.processStartTime };
return (await waitForDaemonExit(identity, { timeoutMs: 0 })).exited;
}
/**
* The stop's own verdict, from what the runtime said about its close. Claim and provider-release
* results come from the shutdown report the close wrote, which the caller merges as it does for any
* graceful stop; what the record adds is the work the close could not finish.
*/
function stopResultFromCompletion(completion: DaemonShutdownCompletion): DaemonStopResult {
// Only a close that both released its registration and finished its work lets this stop vouch for
// what was cleaned up; anything else says the state directory may still be held.
const released = completion.outcome === 'released';
return {
stopped: true,
mode: 'graceful',
cleanupConfidence: released && completion.unfinished.length === 0 ? 'known' : 'unknown',
claimsReleased: [],
claimsOrphaned: [],
claimsSuperseded: [],
claimsUnattributable: [],
providerReleases: { status: 'completed', released: [], pending: [] },
warnings: [
...completion.unfinished,
// A runtime that could not cancel its own work stops serving but keeps its registration and
// lock, because releasing them would let a second daemon drive the same devices.
...(released
? []
: [
'The runtime stopped serving and keeps its registration: a host process still owns it, so nothing outside that process may signal it.',
]),
],
};
}
/**
* A readable record that names a host-owned runtime and no way to reach it: the stop has no instrument
* left. It signals nothing, so it cannot end what it cannot reach, and it says so instead of guessing
* that the record describes a daemon that already left.
*/
function daemonEmbeddedRecordUnaddressableError(paths: DaemonPaths): AppError {
return new AppError(
'COMMAND_FAILED',
`The record on ${paths.baseDir} does not name a transport and a token, so the runtime it describes cannot be reached.`,
{
reason: 'daemon-embedded-stop-unreachable',
stateDir: paths.baseDir,
record: 'incomplete',
hint: 'Nothing was signaled. Inspect the record, or close the host that owns this runtime.',
},
);
}
function daemonShutdownUnconfirmedError(info: DaemonInfo, paths: DaemonPaths): AppError {
return new AppError('COMMAND_FAILED', 'The runtime did not report that it finished leaving.', {
reason: 'daemon-shutdown-unconfirmed',
pid: info.pid,
...(info.processStartTime ? { processStartTime: info.processStartTime } : {}),
stateDir: paths.baseDir,
hint: 'It was not signaled, and it may still be serving. Ask the host that owns it to close it, or retry the stop.',
});
}