Skip to content

Commit f5b44df

Browse files
committed
fix(daemon): judge takeover ownership once inside the retirement
readReusableLocalDaemon rejected an embedded or unknown owner before the central retirement could look at it, so a record whose owner had already exited wedged every managed command instead of recovering. Delete the two client guards and the resolveTimeoutRecovery ownership guard: ownership is judged inside stopAndRetireDaemon on every takeover route (readReusableLocalDaemon, replaceUnreachable, quiesceOrRetireUnresponsive, awaitDrainedDaemon and forceRetireDaemon alike). Map a retained owner reason to the existing typed errors once, in daemonRetirementError for the stop route and daemonTakeoverRetirementError for a takeover, and prove the managed route recovers a crashed embedded or unrecognized owner without signaling it. #3806 (comment)
1 parent 62148c8 commit f5b44df

14 files changed

Lines changed: 265 additions & 179 deletions

File tree

‎packages/daemon-contracts/src/daemon-registration.ts‎

Lines changed: 56 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import {
1515
type DaemonOwnership,
1616
type DaemonPaths,
1717
type DaemonServerMode,
18+
type ObservedDaemonIdentity,
1819
} from './daemon-resolution.ts';
1920

2021
/**
@@ -24,8 +25,8 @@ import {
2425
* however alive it is.
2526
*/
2627
export function readRegisteredDaemonIdentity(infoPath: string): OwnerIdentity | null {
27-
const record = readRegistration(infoPath);
28-
return record.status === 'registered' ? record.identity : null;
28+
const reading = readDaemonRegistration(infoPath);
29+
return reading.status === 'present' ? reading.identity : null;
2930
}
3031

3132
export function processLockHoldsDaemonIdentity(
@@ -46,26 +47,29 @@ type ParsedRegistration = { pid: number | null; startTime: string | null };
4647
type DaemonRecord = Readonly<Record<string, unknown>>;
4748

4849
/**
49-
* Whether `daemon.json` registers a daemon: a projection of the one record reader that adds the
50-
* identity, so a record that names no owner is `ownerless` however it came to.
50+
* The one read of `daemon.json` behind every projection of that record in this module. `identity` is
51+
* null for a record that names no process, and `ownership` is what the record states, read the way a
52+
* reader must read it before anything is signaled.
5153
*/
52-
function readRegistration(
53-
infoPath: string,
54-
):
55-
| Readonly<{ status: 'registered'; identity: OwnerIdentity; record: DaemonRecord }>
56-
| Readonly<{ status: 'absent' | 'unreadable' | 'ownerless' }> {
54+
type DaemonRegistrationReading =
55+
| Readonly<{
56+
status: 'present';
57+
record: DaemonRecord;
58+
identity: OwnerIdentity | null;
59+
ownership: DaemonOwnership;
60+
}>
61+
| Readonly<{ status: 'absent' | 'unreadable' | 'invalid' }>;
62+
63+
function readDaemonRegistration(infoPath: string): DaemonRegistrationReading {
5764
const read = readDaemonRecord(infoPath);
58-
if (read.status !== 'present') {
59-
return { status: read.status === 'invalid' ? 'ownerless' : read.status };
60-
}
65+
if (read.status !== 'present') return { status: read.status };
6166
const parsed = parseRegistration(read.record);
62-
return parsed.pid === null
63-
? { status: 'ownerless' }
64-
: {
65-
status: 'registered',
66-
identity: { pid: parsed.pid, startTime: parsed.startTime },
67-
record: read.record,
68-
};
67+
return {
68+
status: 'present',
69+
record: read.record,
70+
identity: parsed.pid === null ? null : { pid: parsed.pid, startTime: parsed.startTime },
71+
ownership: readDaemonOwnership(read.record.ownership),
72+
};
6973
}
7074

7175
function parseRegistration(parsed: {
@@ -102,15 +106,18 @@ export function readRegisteredDaemonOwnership(
102106
infoPath: string,
103107
owner: OwnerIdentity | null,
104108
): RegisteredDaemonOwnership {
105-
const record = readRegistration(infoPath);
106-
if (record.status !== 'registered') return { state: record.status };
109+
const reading = readDaemonRegistration(infoPath);
110+
if (reading.status !== 'present')
111+
return { state: reading.status === 'invalid' ? 'ownerless' : reading.status };
112+
// A record that names no process describes nothing a caller can own, however it came to that.
113+
if (!reading.identity) return { state: 'ownerless' };
107114
if (!owner) return { state: 'unproven' };
108-
if (ownerIdentityDiffers(record.identity, owner)) {
109-
return { state: 'replaced', identity: record.identity };
115+
if (ownerIdentityDiffers(reading.identity, owner)) {
116+
return { state: 'replaced', identity: reading.identity };
110117
}
111118
// Two missing start times agree, and agreeing on nothing is not ownership: this is the pid-only
112119
// fallback the fence exists to refuse, so it falls to `unproven` rather than to `match`.
113-
if (ownerIdentityMatches(record.identity, owner) && record.identity.startTime !== null) {
120+
if (ownerIdentityMatches(reading.identity, owner) && reading.identity.startTime !== null) {
114121
return { state: 'match' };
115122
}
116123
return { state: 'unproven' };
@@ -152,16 +159,15 @@ export function isUnusableRecord(status: Pick<DaemonStatus, 'state'>): boolean {
152159
*/
153160
export function readDaemonStatus(paths: DaemonPaths): DaemonStatus {
154161
const base = { stateDir: paths.baseDir } as const;
155-
const read = readDaemonRecord(paths.infoPath);
156-
if (read.status !== 'present') {
157-
return read.status === 'absent'
162+
const reading = readDaemonRegistration(paths.infoPath);
163+
if (reading.status !== 'present') {
164+
return reading.status === 'absent'
158165
? { state: 'stopped', ...base }
159-
: { state: read.status, ...base };
166+
: { state: reading.status, ...base };
160167
}
161-
const { record } = read;
162-
const parsed = parseRegistration(record);
163-
if (parsed.pid === null) return { state: 'stopped', ...base };
164-
const identity = { pid: parsed.pid, startTime: parsed.startTime };
168+
const { record } = reading;
169+
if (reading.identity === null) return { state: 'stopped', ...base };
170+
const identity = reading.identity;
165171
if (identity.startTime === null || !isProcessAlive(identity.pid)) {
166172
return { state: 'stopped', ...base };
167173
}
@@ -178,7 +184,7 @@ export function readDaemonStatus(paths: DaemonPaths): DaemonStatus {
178184
state: 'running',
179185
stateDir: paths.baseDir,
180186
pid: identity.pid,
181-
ownership: readDaemonOwnership(record.ownership),
187+
ownership: reading.ownership,
182188
...(typeof record.version === 'string' ? { version: record.version } : {}),
183189
...(transport ? { transport } : {}),
184190
plugins: record.plugins,
@@ -211,16 +217,23 @@ export type DaemonStopIdentity = Readonly<{
211217
* record written before the ownership field named.
212218
*/
213219
export function readDaemonStopIdentity(infoPath: string): DaemonStopIdentity {
214-
const read = readDaemonRecord(infoPath);
215-
if (read.status !== 'present') {
216-
return {
217-
ownership: read.status === 'absent' ? 'absent' : 'unreadable',
218-
identity: null,
219-
};
220-
}
221-
const parsed = parseRegistration(read.record);
222-
const identity = parsed.pid === null ? null : { pid: parsed.pid, startTime: parsed.startTime };
223-
return { ownership: readDaemonOwnership(read.record.ownership), identity };
220+
const reading = readDaemonRegistration(infoPath);
221+
if (reading.status !== 'present')
222+
return { ownership: reading.status === 'absent' ? 'absent' : 'unreadable', identity: null };
223+
return { ownership: reading.ownership, identity: reading.identity };
224+
}
225+
226+
/**
227+
* The observation a retirement judges, from a stop's read of the record. `absent` names no process at
228+
* all, and `unreadable` describes one this read cannot use, so neither travels as an owner; the stop
229+
* route refuses the unreadable record before it gets here.
230+
*/
231+
export function observedDaemonIdentityOfRegistration(
232+
record: DaemonStopIdentity,
233+
): ObservedDaemonIdentity | null {
234+
if (!record.identity || record.ownership === 'absent' || record.ownership === 'unreadable')
235+
return null;
236+
return { ...record.identity, ownership: record.ownership };
224237
}
225238

226239
/**

‎packages/daemon-contracts/src/daemon-stop.ts‎

Lines changed: 51 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,12 @@
11
import { AppError } from '@agent-device/kernel/errors';
22
import type { DaemonRetirementResult } from './daemon-registration-owner.ts';
33

4-
import type { DaemonPaths, ObservedDaemonIdentity } from './daemon-resolution.ts';
4+
import {
5+
DAEMON_LIFECYCLE_REASONS,
6+
DAEMON_RECORD_OWNER_UNRECOGNIZED_REASON,
7+
type DaemonPaths,
8+
type ObservedDaemonIdentity,
9+
} from './daemon-resolution.ts';
510
import type { OwnerIdentity } from '@agent-device/host-kit/process';
611
import type {
712
DaemonShutdownReport,
@@ -43,8 +48,9 @@ export type DaemonStopResult = {
4348
* Retires the daemon whose lifetime the caller read, and shapes that retirement as a stop result.
4449
*
4550
* The caller hands over the lifetime and the owner its own read proved, so routing and retirement judge
46-
* one record. Only a detached daemon reaches here: a host owns an embedded runtime and ends it with its
47-
* own close, and a record this build cannot place proves nothing about what is safe to signal.
51+
* one record. Ownership is judged here and nowhere else: an embedded runtime belongs to its host, and a
52+
* record this build cannot place proves nothing about what is safe to signal, so both come back as the
53+
* refusal that names their owner, with nothing signaled.
4854
*/
4955
export async function stopDaemon(params: {
5056
paths: DaemonPaths;
@@ -95,6 +101,25 @@ export async function stopDaemon(params: {
95101
};
96102
}
97103

104+
/**
105+
* What a retirement that refused because of the record's owner answers with. This is the one place
106+
* those two retained reasons become a stop's own reason: a caller that may not signal this process
107+
* hears the owner's rule, not a retirement that could not be confirmed.
108+
*/
109+
function ownerRefusal(
110+
reason: 'embedded-owner' | 'owner-unproven',
111+
): Readonly<{ reason: string; hint: string }> {
112+
return reason === 'embedded-owner'
113+
? {
114+
reason: DAEMON_LIFECYCLE_REASONS.ownerEmbedded,
115+
hint: 'Nothing was signaled. Close the host that started this runtime, or repair its record after the process has exited.',
116+
}
117+
: {
118+
reason: DAEMON_RECORD_OWNER_UNRECOGNIZED_REASON,
119+
hint: 'Nothing was signaled. Check what owns that process, then repair the record: stop it by its own means and remove its daemon.json so a daemon can register again.',
120+
};
121+
}
122+
98123
function daemonRetirementError(
99124
info: OwnerIdentity,
100125
retirement: Exclude<DaemonRetirementResult, { status: 'retired' }>,
@@ -103,17 +128,29 @@ function daemonRetirementError(
103128
const failure = termination?.status === 'retained' ? termination : undefined;
104129
const error = retirement.status === 'retained' ? retirement.error : undefined;
105130
const { hint, diagnosticId, logPath } = error ?? {};
106-
return new AppError('COMMAND_FAILED', 'Daemon retirement could not be confirmed.', {
107-
pid: info.pid,
108-
processStartTime: info.startTime,
109-
reason: failure ? 'daemon_exit_unconfirmed' : 'daemon_retirement_unconfirmed',
110-
terminationReason: failure?.reason,
111-
signal: failure?.signal,
112-
retirement,
113-
hint,
114-
diagnosticId,
115-
logPath,
116-
});
131+
const refusal =
132+
retirement.status === 'retained' &&
133+
(retirement.reason === 'embedded-owner' || retirement.reason === 'owner-unproven')
134+
? ownerRefusal(retirement.reason)
135+
: undefined;
136+
return new AppError(
137+
'COMMAND_FAILED',
138+
refusal
139+
? 'The daemon was not stopped, because its owner ends it.'
140+
: 'Daemon retirement could not be confirmed.',
141+
{
142+
pid: info.pid,
143+
processStartTime: info.startTime,
144+
reason:
145+
refusal?.reason ?? (failure ? 'daemon_exit_unconfirmed' : 'daemon_retirement_unconfirmed'),
146+
terminationReason: failure?.reason,
147+
signal: failure?.signal,
148+
retirement,
149+
hint: hint ?? refusal?.hint,
150+
diagnosticId,
151+
logPath,
152+
},
153+
);
117154
}
118155

119156
function daemonNotRunningStopResult(): DaemonStopResult {

‎scripts/clean-daemon.ts‎

Lines changed: 6 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,10 @@ import { AppError } from '@agent-device/kernel/errors';
55
import { withCommandHost } from '@agent-device/host-kit/command';
66
import { captureHostEnvironment } from '@agent-device/host-kit/process';
77
import { resolveDaemonPaths } from '@agent-device/daemon-contracts/daemon-resolution';
8-
import { readDaemonStopIdentity } from '@agent-device/daemon-contracts/daemon-registration';
8+
import {
9+
observedDaemonIdentityOfRegistration,
10+
readDaemonStopIdentity,
11+
} from '@agent-device/daemon-contracts/daemon-registration';
912
import {
1013
stopAndRetireDaemon,
1114
recoverAbandonedDaemonRegistration,
@@ -22,7 +25,7 @@ const paths = resolveDaemonPaths(process.env.AGENT_DEVICE_STATE_DIR, {
2225
});
2326
const retirement = await stopAndRetireDaemon({
2427
paths,
25-
observed: readCleanupIdentity(paths.infoPath),
28+
observed: observedDaemonIdentityOfRegistration(readDaemonStopIdentity(paths.infoPath)),
2629
mode: 'graceful',
2730
termTimeoutMs: DAEMON_TERM_TIMEOUT_MS,
2831
killTimeoutMs: DAEMON_KILL_TIMEOUT_MS,
@@ -65,7 +68,7 @@ async function pruneStaleDevStateDirs(): Promise<void> {
6568
const paths = resolveDaemonPaths(dirPath, { cwd: process.cwd() });
6669
const result = await recoverAbandonedDaemonRegistration({
6770
paths,
68-
observed: readCleanupIdentity(paths.infoPath),
71+
observed: observedDaemonIdentityOfRegistration(readDaemonStopIdentity(paths.infoPath)),
6972
});
7073
if (result.status === 'retired')
7174
process.stdout.write(
@@ -107,10 +110,3 @@ function statMtimeMs(filePath: string): number {
107110
return 0;
108111
}
109112
}
110-
111-
function readCleanupIdentity(infoPath: string) {
112-
const { identity, ownership } = readDaemonStopIdentity(infoPath);
113-
return identity && ownership !== 'absent' && ownership !== 'unreadable'
114-
? { ...identity, ownership }
115-
: null;
116-
}

‎src/__tests__/daemon-registration-embedded-owner.test.ts‎

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ import { stopAndRetireDaemon } from '@agent-device/daemon-contracts/daemon-regis
77
import { resolveDaemonPaths } from '@agent-device/daemon-contracts/daemon-resolution';
88
import { stopRecordedDaemon } from '../daemon-client/daemon-client-stop.ts';
99
import { readDaemonStopIdentity } from '@agent-device/daemon-contracts/daemon-registration';
10-
import { createTestHostEnvironment, mkdtempForTestSync } from '@agent-device/test-kit';
10+
import { mkdtempForTestSync } from '@agent-device/test-kit';
1111

1212
// What the registration owner may do to a record a host process owns: prove the lifetime is gone
1313
// before removing anything, and never signal the process the record names.
@@ -100,10 +100,7 @@ test('retirement of an embedded record proves the lifetime is gone and signals n
100100
}
101101

102102
await assert.rejects(
103-
stopRecordedDaemon(readDaemonStopIdentity(paths.infoPath), {
104-
paths,
105-
host: createTestHostEnvironment(),
106-
}),
103+
stopRecordedDaemon(readDaemonStopIdentity(paths.infoPath), paths),
107104
(error: unknown) =>
108105
error instanceof AppError &&
109106
error.details?.reason === 'daemon-owner-embedded' &&
@@ -185,10 +182,7 @@ test('retirement refuses to signal a record whose owner it cannot recognize', as
185182
assert.equal(result.reason, 'owner-unproven', ownership);
186183
if (ownership === 'sideways')
187184
await assert.rejects(
188-
stopRecordedDaemon(readDaemonStopIdentity(paths.infoPath), {
189-
paths,
190-
host: createTestHostEnvironment(),
191-
}),
185+
stopRecordedDaemon(readDaemonStopIdentity(paths.infoPath), paths),
192186
(error: unknown) =>
193187
error instanceof AppError && error.details?.reason === 'daemon-owner-unrecognized',
194188
);

0 commit comments

Comments
 (0)