Skip to content

Commit 36d615a

Browse files
committed
fix: recover monitored daemon birth from owned registration
1 parent 0d1c7b3 commit 36d615a

4 files changed

Lines changed: 175 additions & 30 deletions

File tree

‎src/__tests__/daemon-registration-owner.test.ts‎

Lines changed: 49 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -443,8 +443,10 @@ for (const [failure, reason, expectedSignals] of privateStartupFailures) {
443443
await waitForFixtureFile(paths.infoPath);
444444
actualStartTime = hostProcess.readProcessStartTime(launch.pid);
445445
assert.ok(actualStartTime);
446-
if (failure === 'missing-birth') assert.equal(launch.startTime, undefined);
447-
else assert.ok(launch.startTime);
446+
if (failure === 'missing-birth') {
447+
assert.equal(launch.startTime, undefined);
448+
replaceInfo(paths, launch.pid);
449+
} else assert.ok(launch.startTime);
448450
const metadata = fs.readFileSync(paths.infoPath, 'utf8');
449451
let joined = false;
450452
void launch.exited.then(() => {
@@ -516,7 +518,7 @@ function assertPrivateStartupFailureDetails(
516518
}
517519
}
518520

519-
test('private startup completion stays bounded without birth proof and retains the live child', async () => {
521+
test('private startup completion stays bounded without published birth proof and retains the live child', async () => {
520522
const ownedStateDir = createOwnedReplayStateDir();
521523
const paths = ownedStateDir.paths;
522524
const birthProbe = vi.spyOn(hostProcess, 'readProcessStartTime').mockReturnValueOnce(null);
@@ -533,6 +535,7 @@ test('private startup completion stays bounded without birth proof and retains t
533535
actualStartTime = hostProcess.readProcessStartTime(launch.pid);
534536
assert.ok(actualStartTime);
535537
assert.equal(launch.startTime, undefined);
538+
replaceInfo(paths, launch.pid);
536539
const metadata = fs.readFileSync(paths.infoPath, 'utf8');
537540
const signals = vi.spyOn(process, 'kill');
538541
const pending = stopAndRetireDaemon({
@@ -787,3 +790,46 @@ for (const mode of ['graceful', 'forced'] as const) {
787790
}
788791
});
789792
}
793+
794+
test.for(['different pid', 'different birth', 'joined child'] as const)(
795+
'startup birth recovery refuses $0 despite published metadata',
796+
async (proof) => {
797+
const ownedStateDir = createOwnedReplayStateDir();
798+
const paths = ownedStateDir.paths;
799+
const probe = vi.spyOn(hostProcess, 'readProcessStartTime').mockReturnValueOnce(null);
800+
const launch = launchDaemonProcess({
801+
paths,
802+
args: registeredDaemonFixtureArgs(paths, fields),
803+
serverMode: 'socket',
804+
ownedStateDir,
805+
});
806+
probe.mockRestore();
807+
let actualStartTime: string | null = null;
808+
try {
809+
await waitForFixtureFile(paths.infoPath);
810+
actualStartTime = hostProcess.readProcessStartTime(launch.pid);
811+
assert.ok(actualStartTime);
812+
if (proof === 'joined child') {
813+
const metadata = fs.readFileSync(paths.infoPath);
814+
const savedLock = path.join(paths.baseDir, 'saved-lock');
815+
fs.cpSync(paths.lockPath, savedLock, { recursive: true });
816+
process.kill(launch.pid, 'SIGTERM');
817+
await launch.exited;
818+
fs.writeFileSync(paths.infoPath, metadata);
819+
fs.cpSync(savedLock, paths.lockPath, { recursive: true });
820+
} else {
821+
replaceInfo(
822+
paths,
823+
proof === 'different pid' ? process.pid : launch.pid,
824+
proof === 'different birth' ? 'different-birth' : actualStartTime,
825+
);
826+
}
827+
const metadata = fs.readFileSync(paths.infoPath, 'utf8');
828+
assert.deepEqual(launch.readIdentity(), { pid: launch.pid, startTime: null });
829+
assert.equal(launch.startTime, undefined);
830+
assert.equal(fs.readFileSync(paths.infoPath, 'utf8'), metadata);
831+
} finally {
832+
await finishPrivateTestDaemons(paths, { ...launch, startTime: actualStartTime ?? undefined });
833+
}
834+
},
835+
);

‎src/daemon-client/__tests__/daemon-client-startup-race.test.ts‎

Lines changed: 64 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,11 @@ import path from 'node:path';
44
import { afterEach, test, vi } from 'vitest';
55
import { AppError } from '@agent-device/kernel/errors';
66
import { tryAcquireProcessLock, inspectProcessLock } from '@agent-device/host-kit/file';
7-
import { readCurrentOwnerIdentity, isProcessAlive } from '@agent-device/host-kit/process';
7+
import {
8+
readCurrentOwnerIdentity,
9+
isProcessAlive,
10+
readProcessStartTime,
11+
} from '@agent-device/host-kit/process';
812
import { readVersion } from '@agent-device/host-kit/version';
913
import { mkdtempForTestSync } from '../../__tests__/test-utils/tmp-dir.ts';
1014
import {
@@ -18,12 +22,17 @@ import {
1822
import { closeLoopbackServer, supportsLoopbackBind } from '../../__tests__/test-utils/loopback.ts';
1923
import { resolveDaemonPaths, type DaemonPaths } from '../../daemon-resolution.ts';
2024
import { sendToDaemon } from '../daemon-client.ts';
25+
import * as lifecycle from '../daemon-client-lifecycle.ts';
2126
import { DAEMON_STARTUP_EXIT_CODES } from '../../daemon-registration-owner.ts';
2227

2328
vi.mock('@agent-device/host-kit/command', async (importOriginal) => ({
2429
...(await importOriginal<typeof import('@agent-device/host-kit/command')>()),
2530
runCmdDetachedMonitored: vi.fn(),
2631
}));
32+
vi.mock('@agent-device/host-kit/process', async (importOriginal) => {
33+
const actual = await importOriginal<typeof import('@agent-device/host-kit/process')>();
34+
return { ...actual, readProcessStartTime: vi.fn(actual.readProcessStartTime) };
35+
});
2736
vi.mock('@agent-device/host-kit/retry', async (importOriginal) => ({
2837
...(await importOriginal<typeof import('@agent-device/host-kit/retry')>()),
2938
sleep: vi.fn(),
@@ -572,3 +581,57 @@ test('a failed own transport probe retires and joins the private startup before
572581
await closeLoopbackServer(http.server);
573582
}
574583
});
584+
585+
test.for(['default', 'private replay'] as const)(
586+
'a missed initial birth probe recovers the monitored child for $0',
587+
async (route, t) => {
588+
if (!(await supportsLoopbackBind())) return t.skip('loopback unavailable');
589+
const http = await startHttpDaemonFixture({ devices: [], sessionActive: false });
590+
const defaultPaths = resolveDaemonPaths(mkdtempForTestSync('daemon-missed-birth-'));
591+
const ensure = vi.spyOn(lifecycle, 'ensureDaemon');
592+
let paths: DaemonPaths | undefined;
593+
let child: ReturnType<typeof runCmdDetachedMonitored> | undefined;
594+
let now = Date.now();
595+
vi.spyOn(Date, 'now').mockImplementation(() => now);
596+
spawn.mockImplementation((_command, _args, options) => {
597+
paths = resolveDaemonPaths(String(options?.env?.AGENT_DEVICE_STATE_DIR));
598+
child = spawnRegisteredDaemonFixture(paths, fields(http.port), options);
599+
vi.mocked(readProcessStartTime).mockReturnValueOnce(null);
600+
return child;
601+
});
602+
pause.mockImplementation(async () => {
603+
assert.ok(paths);
604+
if (http.rpcRequests.length === 0) {
605+
await awaitFile(paths.infoPath);
606+
now += 1_000;
607+
} else {
608+
await actualRetry.sleep(10);
609+
now += 10;
610+
}
611+
});
612+
try {
613+
const response = await sendToDaemon(
614+
route === 'default'
615+
? request(defaultPaths)
616+
: { session: 'default', command: 'replay', positionals: [], flags: {} },
617+
);
618+
assert.equal(response.ok, true);
619+
assert.equal(ensure.mock.calls.length, 1);
620+
assert.equal((await ensure.mock.results[0]!.value).startedByClient, true);
621+
assert.equal(spawn.mock.calls.length, 1);
622+
assert.equal(http.rpcRequests.length, 1);
623+
assert.ok(paths && child);
624+
if (route === 'private replay') {
625+
await child.exited;
626+
assert.equal(isProcessAlive(child.pid), false);
627+
assert.equal(fs.existsSync(paths.baseDir), false);
628+
} else {
629+
assert.equal(isProcessAlive(child.pid), true);
630+
assert.equal(inspectProcessLock(paths.lockPath).state, 'held');
631+
assert.equal(fs.existsSync(paths.infoPath), true);
632+
}
633+
} finally {
634+
await closeLoopbackServer(http.server);
635+
}
636+
},
637+
);

‎src/daemon-client/daemon-client-lifecycle.ts‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -466,7 +466,7 @@ async function retireStartupAttempt(
466466
): Promise<{ cleanup: DaemonRetirementResult; joined: boolean }> {
467467
const cleanup = await stopAndRetireDaemon({
468468
paths: settings.paths,
469-
observed: { pid: launch.pid, startTime: launch.startTime ?? null },
469+
observed: launch.readIdentity(),
470470
mode: 'graceful',
471471
ownedStateDir: settings.ownedStateDir,
472472
termTimeoutMs: Math.min(3_000, remainingStartupBudget(deadline)),
@@ -801,10 +801,11 @@ async function readReadyLaunchedDaemon(
801801

802802
/** Whether `info` names the daemon process this client launched: same pid and start time. */
803803
function isLaunchedDaemon(info: DaemonInfo, launch: DaemonStartupLaunch): boolean {
804+
const identity = launch.readIdentity();
804805
return (
805-
info.pid === launch.pid &&
806-
launch.startTime !== undefined &&
807-
info.processStartTime === launch.startTime
806+
info.pid === identity.pid &&
807+
identity.startTime !== null &&
808+
info.processStartTime === identity.startTime
808809
);
809810
}
810811

‎src/daemon-registration-owner.ts‎

Lines changed: 57 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,13 @@ import {
1010
} from './daemon-process.ts';
1111
import {
1212
readCurrentOwnerIdentity,
13+
ownerIdentityMatches,
1314
readProcessStartTime,
1415
type OwnerIdentity,
1516
} from '@agent-device/host-kit/process';
1617
import {
1718
publishFileSync,
19+
inspectProcessLock,
1820
tryAcquireProcessLock,
1921
acquireProcessLockAcquisition,
2022
type ProcessLockAttempt,
@@ -30,6 +32,7 @@ import {
3032
import { findUnrecoveredRepairCommitFailure } from './session-repair-tombstone.ts';
3133
import {
3234
readRegisteredDaemonOwnership,
35+
readRegisteredDaemonIdentity,
3336
type RegisteredDaemonOwnership,
3437
} from './daemon-registration.ts';
3538
import {
@@ -182,8 +185,9 @@ export type DaemonStartupLaunch = Readonly<{
182185
pid: number;
183186
startTime?: string;
184187
exited: Promise<ExecDetachedExit>;
188+
readIdentity(): OwnerIdentity;
185189
}>;
186-
type OwnedStartup = { launch: DaemonStartupLaunch; joined: boolean };
190+
type OwnedStartup = { launch: DaemonStartupLaunch; joined: boolean; startTime?: string };
187191
type PrivateReplayState = {
188192
paths: Readonly<DaemonPaths>;
189193
startups: OwnedStartup[];
@@ -227,23 +231,40 @@ export function launchDaemonProcess(
227231
},
228232
stdio: ['ignore', logFd, logFd],
229233
});
230-
const startup: OwnedStartup = { launch: Object.freeze({ ...monitored }), joined: false };
231-
if (owned) {
232-
owned.startups.push(startup);
233-
void monitored.exited.then(() => {
234-
startup.joined = true;
235-
});
236-
}
237-
startup.launch = Object.freeze({
238-
...startup.launch,
239-
startTime: readProcessStartTime(monitored.pid) ?? undefined,
234+
const boundPaths = { ...input.paths };
235+
const startup: OwnedStartup = {
236+
launch: Object.freeze({
237+
...monitored,
238+
get startTime() {
239+
return startup.startTime;
240+
},
241+
readIdentity() {
242+
recoverStartupBirth(boundPaths, startup);
243+
return { pid: monitored.pid, startTime: startup.startTime ?? null };
244+
},
245+
}),
246+
joined: false,
247+
};
248+
if (owned) owned.startups.push(startup);
249+
void monitored.exited.then(() => {
250+
startup.joined = true;
240251
});
252+
startup.startTime = readProcessStartTime(monitored.pid) ?? undefined;
241253
return startup.launch;
242254
} finally {
243255
fs.closeSync(logFd);
244256
}
245257
}
246258

259+
function recoverStartupBirth(paths: DaemonPaths, startup: OwnedStartup): void {
260+
if (startup.startTime || startup.joined) return;
261+
const registration = readRegisteredDaemonIdentity(paths.infoPath);
262+
if (registration?.pid !== startup.launch.pid || !registration.startTime) return;
263+
const lock = inspectProcessLock(paths.lockPath);
264+
if (lock.state !== 'held' || !ownerIdentityMatches(lock.owner, registration)) return;
265+
startup.startTime = registration.startTime;
266+
}
267+
247268
function requirePrivateReplayState(
248269
capability: OwnedReplayStateDir,
249270
paths: DaemonPaths,
@@ -309,20 +330,12 @@ export async function stopAndRetireDaemon(
309330
}>,
310331
): Promise<DaemonRetirementResult> {
311332
let owned: PrivateReplayState | undefined;
333+
let observed = input.observed;
312334
try {
313335
if (input.ownedStateDir) {
314336
owned = requirePrivateReplayState(input.ownedStateDir, input.paths);
315337
owned.sealed = true;
316-
const launch = owned.startups.find(
317-
({ launch }) =>
318-
launch.pid === input.observed?.pid && launch.startTime === input.observed?.startTime,
319-
)?.launch;
320-
if (!launch?.startTime && (input.observed || owned.startups.length > 0))
321-
throw new AppError(
322-
'COMMAND_FAILED',
323-
'The observed daemon is not an owned startup lifetime.',
324-
{ reason: 'daemon_private_startup_unowned' },
325-
);
338+
observed = confirmOwnedStartupIdentity(owned, observed);
326339
if (owned.retirement) return await owned.retirement;
327340
}
328341
} catch (error) {
@@ -335,7 +348,7 @@ export async function stopAndRetireDaemon(
335348
};
336349
}
337350
const retirement = retireObservedDaemon(
338-
input,
351+
{ ...input, observed },
339352
(identity) =>
340353
stopDaemonProcess(identity, {
341354
mode: input.mode,
@@ -352,6 +365,28 @@ export async function stopAndRetireDaemon(
352365
return result;
353366
}
354367

368+
function confirmOwnedStartupIdentity(
369+
owned: PrivateReplayState,
370+
observed: OwnerIdentity | null,
371+
): OwnerIdentity | null {
372+
if (!observed) {
373+
if (owned.startups.length === 0) return null;
374+
throw unownedStartupError();
375+
}
376+
const startup = owned.startups.find(({ launch }) => launch.pid === observed.pid);
377+
if (!startup) throw unownedStartupError();
378+
const { launch } = startup;
379+
const identity = observed.startTime ? observed : launch.readIdentity();
380+
if (launch.startTime && launch.startTime === identity.startTime) return identity;
381+
throw unownedStartupError();
382+
}
383+
384+
function unownedStartupError(): AppError {
385+
return new AppError('COMMAND_FAILED', 'The observed daemon is not an owned startup lifetime.', {
386+
reason: 'daemon_private_startup_unowned',
387+
});
388+
}
389+
355390
/** Recovers a confirmed abandoned registration without signaling a live process. */
356391
export async function recoverAbandonedDaemonRegistration(
357392
input: DaemonRetirementInput,

0 commit comments

Comments
 (0)