Repository navigation
Expand file tree
/
Copy pathdaemon-client-stop.ts
More file actions
200 lines (190 loc) · 8.84 KB
/
Copy pathdaemon-client-stop.ts
File metadata and controls
200 lines (190 loc) · 8.84 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
import {
daemonShutdownCompletionAnswers,
readDaemonShutdownCompletion,
type DaemonShutdownCompletion,
type DaemonShutdownCompletionQuery,
} from '@agent-device/daemon-contracts/daemon-shutdown-report';
import { waitForDaemonExit } from '@agent-device/daemon-contracts/daemon-process';
import {
DAEMON_LIFECYCLE_REASONS,
readDaemonLifecycleCensus,
type DaemonPaths,
} from '@agent-device/daemon-contracts/daemon-resolution';
import {
daemonNotRunningStopResult,
type DaemonStopResult,
} from '@agent-device/daemon-contracts/daemon-stop';
import { AppError } from '@agent-device/kernel/errors';
import { sleep } from '@agent-device/host-kit/retry';
import {
readDaemonRecordAddress,
type DaemonInfo,
type DaemonRecordAddress,
} from './daemon-client-metadata.ts';
import {
DAEMON_LEAVE_WAIT_MS,
requestDaemonLifecycle,
type DaemonLifecycleTarget,
} from './daemon-client-lifecycle-request.ts';
// How long a stop keeps asking while the runtime refuses because it is still starting up and has no
// drain path to run yet.
const EMBEDDED_STOP_ASK_BUDGET_MS = 5_000;
const EMBEDDED_STOP_ASK_GAP_MS = 250;
const EMBEDDED_STOP_POLL_MS = 100;
// How often the wait asks whether the runtime's host is still alive. A lifetime read costs a process
// table lookup, and the answer changes on a much slower clock than the record does.
const EMBEDDED_STOP_LIFETIME_POLL_MS = 1_000;
/**
* Stops the embedder-owned runtime registered on a state directory and reports what it answered.
*
* Nothing here signals a process. A host owns this runtime, so the only stop available from outside is
* the authenticated shutdown request the runtime answers itself, followed by the bounded completion
* record it leaves when its close has a report to give.
*/
export async function stopEmbeddedDaemon(target: DaemonLifecycleTarget): Promise<DaemonStopResult> {
const deadlineMs = Date.now() + DAEMON_LEAVE_WAIT_MS;
// Only a state directory with no record at all is a place where nothing runs. A record that cannot
// be read as one says something holds this directory and this stop cannot address it, which is a
// refusal, not an absence: reporting `not-running` there would tell the operator the runtime is
// gone while its host may be serving, and its registration still in place.
const record = readDaemonRecordAddress(target.paths.infoPath);
if (record.status === 'absent') return daemonNotRunningStopResult();
if (record.status !== 'addressable') {
throw daemonEmbeddedRecordUnaddressableError(target.paths, record.status);
}
const info = record.info;
// An answer of `gone` is waited on the same way as an agreement: a daemon that stopped answering has
// either left, is leaving, or was replaced, and the two reads below are what tells those apart.
await askToLeave(info, target, Date.now() + EMBEDDED_STOP_ASK_BUDGET_MS);
const waited = await waitForClose(target.paths, instanceOf(info), deadlineMs);
if (waited === 'instance-gone') return daemonNotRunningStopResult();
if (!waited) throw daemonShutdownUnconfirmedError(info, target.paths);
return stopResultFromCompletion(waited);
}
/** The instance a stop addresses: the record's own lifetime, which is what a record is matched on. */
function instanceOf(info: DaemonInfo): DaemonShutdownCompletionQuery {
return { pid: info.pid, processStartTime: info.processStartTime ?? '' };
}
/**
* Sends the shutdown request, asking again while the runtime refuses because it is starting up. What it
* answers is not this function's business: an agreement, a refusal that the daemon is already leaving,
* and a connection that cannot be made all send the stop to the same wait for evidence.
*/
async function askToLeave(
info: DaemonInfo,
target: DaemonLifecycleTarget,
askUntilMs: number,
): Promise<void> {
for (;;) {
try {
await requestDaemonLifecycle(info, target, 'shutdown');
return;
} catch (error) {
// A runtime that is still starting has no drain path and says so with its census. That is the
// one refusal worth asking again about: it will be able to drain in a moment.
if (!refusedWhileStartingUp(error) || Date.now() >= askUntilMs) throw error;
}
await sleep(EMBEDDED_STOP_ASK_GAP_MS);
}
}
function refusedWhileStartingUp(error: unknown): boolean {
return (
error instanceof AppError &&
error.details?.reason === DAEMON_LIFECYCLE_REASONS.busy &&
readDaemonLifecycleCensus(error.details)?.startingUp === true
);
}
/**
* Reads the record the closing runtime publishes, and notices when its writer can no longer publish
* one: the instance's own lifetime is provably over, so what is on disk is the whole truth and nothing
* is left to wait for.
*
* Both reads are needed. The record answers by instance rather than by request, so a second stop of the
* same runtime reads the same answer instead of sitting out its budget for a record written for
* somebody else's request. And the lifetime check answers the case the record cannot: a host that died
* between removing the registration and writing its report would otherwise leave a stop waiting on a
* writer that is gone.
*/
async function waitForClose(
paths: DaemonPaths,
instance: DaemonShutdownCompletionQuery,
deadlineMs: number,
): Promise<DaemonShutdownCompletion | 'instance-gone' | null> {
let nextLifetimeCheckMs = 0;
while (Date.now() < deadlineMs) {
const completion = readDaemonShutdownCompletion(paths.baseDir);
if (completion && daemonShutdownCompletionAnswers(completion, instance)) return completion;
if (Date.now() >= nextLifetimeCheckMs) {
nextLifetimeCheckMs = Date.now() + EMBEDDED_STOP_LIFETIME_POLL_MS;
if (await instanceEnded(instance)) return 'instance-gone';
}
await sleep(Math.min(EMBEDDED_STOP_POLL_MS, Math.max(0, deadlineMs - Date.now())));
}
return null;
}
async function instanceEnded(instance: DaemonShutdownCompletionQuery): Promise<boolean> {
if (!instance.processStartTime.trim()) return false;
const identity = { pid: instance.pid, startTime: instance.processStartTime };
return (await waitForDaemonExit(identity, { timeoutMs: 0 })).exited;
}
/**
* The stop's own verdict, from what the runtime said about its close. Claim and provider-release
* results come from the shutdown report the close wrote, which the caller merges as it does for any
* graceful stop; what the record adds is the work the close could not finish.
*/
function stopResultFromCompletion(completion: DaemonShutdownCompletion): DaemonStopResult {
// Only a close that both released its registration and finished its work lets this stop vouch for
// what was cleaned up; anything else says the state directory may still be held.
const released = completion.outcome === 'released';
return {
stopped: true,
mode: 'graceful',
cleanupConfidence: released && completion.unfinished.length === 0 ? 'known' : 'unknown',
claimsReleased: [],
claimsOrphaned: [],
claimsSuperseded: [],
claimsUnattributable: [],
providerReleases: { status: 'completed', released: [], pending: [] },
warnings: [
...completion.unfinished,
// A runtime that could not cancel its own work stops serving but keeps its registration and
// lock, because releasing them would let a second daemon drive the same devices.
...(released
? []
: [
'The runtime stopped serving and keeps its registration: a host process still owns it, so nothing outside that process may signal it.',
]),
],
};
}
/**
* A record that states a host-owned runtime and cannot be addressed: the stop has no instrument left.
* It signals nothing, so it cannot end what it cannot reach, and it says so instead of guessing that
* an unreadable record describes a daemon that already left.
*/
function daemonEmbeddedRecordUnaddressableError(
paths: DaemonPaths,
status: Exclude<DaemonRecordAddress['status'], 'addressable' | 'absent'>,
): AppError {
return new AppError(
'COMMAND_FAILED',
status === 'unreadable'
? `The record on ${paths.baseDir} cannot be read, so the runtime it describes cannot be asked to leave.`
: `The record on ${paths.baseDir} does not name a transport and a token, so the runtime it describes cannot be reached.`,
{
reason: 'daemon_embedded_stop_unreachable',
stateDir: paths.baseDir,
record: status,
hint: 'Nothing was signaled. Inspect the record, or close the host that owns this runtime.',
},
);
}
function daemonShutdownUnconfirmedError(info: DaemonInfo, paths: DaemonPaths): AppError {
return new AppError('COMMAND_FAILED', 'The runtime did not report that it finished leaving.', {
reason: 'daemon_shutdown_unconfirmed',
pid: info.pid,
...(info.processStartTime ? { processStartTime: info.processStartTime } : {}),
stateDir: paths.baseDir,
hint: 'It was not signaled, and it may still be serving. Ask the host that owns it to close it, or retry the stop.',
});
}