Skip to content

Commit ca6e0af

Browse files
committed
fix(runtime): preserve platform owner shutdown failures
1 parent 144bd59 commit ca6e0af

5 files changed

Lines changed: 212 additions & 16 deletions

File tree

‎packages/daemon/src/server/http/daemon-runtime.ts‎

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -776,8 +776,8 @@ async function startDaemonRuntimeOnHost(
776776
});
777777

778778
let abandonedRequests = 0;
779-
let platformOwnersPending = true;
780-
const hasUnfinishedWork = () => abandonedRequests > 0 || platformOwnersPending;
779+
let platformOwners: ShutdownReport['platformOwners'] = 'pending';
780+
const hasUnfinishedWork = () => abandonedRequests > 0 || platformOwners !== 'closed';
781781
let shutdownOutcome: Parameters<DaemonRegistrationOwner['finish']>[0];
782782
const drainInFlightRequests = async (): Promise<ShutdownReport['requests']> => {
783783
const total = inFlightRequests.size;
@@ -864,8 +864,18 @@ async function startDaemonRuntimeOnHost(
864864
() => deviceRuntimeGateway.shutdown(),
865865
options.shutdownBudgets?.ownerMs ?? DAEMON_PLATFORM_OWNER_SHUTDOWN_TIMEOUT_MS,
866866
);
867-
if (ownerClose.kind === 'rejected') throw ownerClose.error;
868-
platformOwnersPending = ownerClose.kind === 'deadline';
867+
platformOwners =
868+
ownerClose.kind === 'rejected'
869+
? 'failed'
870+
: ownerClose.kind === 'deadline'
871+
? 'pending'
872+
: 'closed';
873+
if (ownerClose.kind === 'rejected') {
874+
await emitDaemonDiagnostic(logPath, 'daemon_shutdown_platform_owner_failed', {
875+
error:
876+
ownerClose.error instanceof Error ? ownerClose.error.message : String(ownerClose.error),
877+
});
878+
}
869879
const plugins = await pluginClose.close();
870880
for (const failure of plugins.failed) {
871881
await emitDaemonDiagnostic(logPath, 'provider_plugin_cleanup_failed', {
@@ -885,7 +895,7 @@ async function startDaemonRuntimeOnHost(
885895
unknown: [...unknownLeaseIds],
886896
},
887897
requests,
888-
platformOwners: platformOwnersPending ? 'pending' : 'closed',
898+
platformOwners,
889899
runners: { stopped: [...stoppedRunners], handedOff: [...handedOffRunners] },
890900
sessions,
891901
plugins,

‎packages/daemon/src/server/http/server-shutdown.ts‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ export type ShutdownReport = {
6464
*/
6565
requests: { drained: number; abandoned: number };
6666
sessions: { closed: number; failed: number };
67-
platformOwners: 'closed' | 'pending';
67+
platformOwners: 'closed' | 'pending' | 'failed';
6868
/** `pending` plugins were still shutting down when the budget ran out. */
6969
plugins: {
7070
disposed: string[];
@@ -94,6 +94,11 @@ export function describeUnfinishedShutdownWork(report: ShutdownReport): string[]
9494
'Daemon shutdown: platform owners were still shutting down when the budget ran out; the daemon registration stays held until this process exits.',
9595
);
9696
}
97+
if (report.platformOwners === 'failed') {
98+
lines.push(
99+
'Daemon shutdown: platform owner shutdown failed; the daemon registration stays held until this process exits.',
100+
);
101+
}
97102
for (const provider of report.plugins.pending) {
98103
lines.push(
99104
`Daemon shutdown: plugin ${provider} was still shutting down when the budget ran out.`,

‎src/platform-runtime-gateway.test.ts‎

Lines changed: 157 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,18 @@ import { screenshotRuntimeOperationFacts } from '@agent-device/contracts/screens
1414
import { viewportRuntimeOperationFacts } from '@agent-device/contracts/viewport-runtime';
1515
import type { DeviceInfo } from '@agent-device/kernel/device';
1616
import { createLimrunRuntime } from '@agent-device/provider-limrun';
17-
import { describe, expect, test, vi } from 'vitest';
17+
import { describe, expect, expectTypeOf, test, vi } from 'vitest';
18+
import type { AgentDeviceHost } from './sdk/host-types.ts';
19+
import fs from 'node:fs';
20+
import { resolveDaemonPaths } from '@agent-device/daemon-contracts/daemon-resolution';
21+
import { createTestHostEnvironment, mkdtempForTestSync } from '@agent-device/test-kit';
22+
import { startDaemonRuntime } from '../packages/daemon/src/server/http/daemon-runtime.ts';
23+
import { describeUnfinishedShutdownWork } from '../packages/daemon/src/server/http/server-shutdown.ts';
24+
import {
25+
applicationLifecycleResourceFixture,
26+
daemonRuntimeDepsFixture,
27+
platformOwnerLifecycleFixture,
28+
} from '../packages/daemon/src/__tests__/support/daemon-runtime-deps-fixture.ts';
1829
import { createComposedPlatformRuntimeGateway } from './platform-runtime-gateway.ts';
1930
import {
2031
gatewayFixture as gateway,
@@ -28,6 +39,151 @@ import {
2839
} from './platform-runtime-gateway.fixtures.ts';
2940

3041
describe('composed platform runtime gateway', () => {
42+
test('the SDK close report exposes each platform owner outcome', () => {
43+
expectTypeOf<Awaited<ReturnType<AgentDeviceHost['close']>>['platformOwners']>().toEqualTypeOf<
44+
'closed' | 'pending' | 'failed'
45+
>();
46+
});
47+
48+
test('shutdown waits for an owner load already in progress', async () => {
49+
const ref = providerRuntimeOwner('limrun', 'loading');
50+
const shutdown = vi.fn(async () => {});
51+
let finishLoad!: () => void;
52+
const loadReady = new Promise<void>((resolve) => {
53+
finishLoad = resolve;
54+
});
55+
const registration = providerRuntime({
56+
ref,
57+
load: async () => {
58+
await loadReady;
59+
return { ...runtimeOwner({ ref }), shutdown };
60+
},
61+
});
62+
const runtimeGateway = gateway([registration]);
63+
const binding = runtimeGateway.bind({
64+
device,
65+
intent: { kind: 'exact-owner', owner: ref, fence: { token: 'fence', generation: 1 } },
66+
scope,
67+
});
68+
const closing = runtimeGateway.shutdown();
69+
finishLoad();
70+
await binding;
71+
await closing;
72+
expect(shutdown).toHaveBeenCalledOnce();
73+
});
74+
test('daemon reports a composed owner failure and retains registration while completing cleanup', async () => {
75+
const ref = providerRuntimeOwner('limrun', 'failed');
76+
const shutdown = vi.fn(async () => {
77+
throw new Error('owner refused');
78+
});
79+
const registration = providerRuntime({
80+
ref,
81+
load: async () => ({ ...runtimeOwner({ ref }), shutdown }),
82+
});
83+
const composed = gateway([registration]);
84+
await composed.bind({
85+
device,
86+
intent: { kind: 'exact-owner', owner: ref, fence: { token: 'fence', generation: 1 } },
87+
scope,
88+
});
89+
const stateDir = mkdtempForTestSync('agent-device-owner-failure-');
90+
const paths = resolveDaemonPaths(stateDir);
91+
const finalize = vi.fn(async () => ['runner']);
92+
const pluginClose = vi.fn(async () => []);
93+
const clearLock = vi.fn(async () => {});
94+
const runtime = await startDaemonRuntime({
95+
host: createTestHostEnvironment({
96+
env: {
97+
...process.env,
98+
AGENT_DEVICE_STATE_DIR: stateDir,
99+
AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: '0',
100+
AGENT_DEVICE_DAEMON_SERVER_MODE: 'http',
101+
},
102+
}),
103+
stderr: { write: () => {} },
104+
stdout: { write: () => {} },
105+
composeDeps: daemonRuntimeDepsFixture({
106+
deviceRuntimeGateway: {
107+
...composed,
108+
applicationLifecycle: {
109+
...applicationLifecycleResourceFixture(),
110+
finalizeDaemonShutdown: finalize,
111+
},
112+
},
113+
shutdownProviderRuntimes: pluginClose,
114+
ownerLifecycle: platformOwnerLifecycleFixture({ clearDaemonLockConfiguration: clearLock }),
115+
}),
116+
});
117+
const report = await runtime.close();
118+
expect(report.platformOwners).toBe('failed');
119+
expect(report.runners.stopped).toEqual(['runner']);
120+
expect(describeUnfinishedShutdownWork(report)).toEqual([
121+
expect.stringContaining('platform owner shutdown failed'),
122+
]);
123+
expect(fs.readFileSync(paths.logPath, 'utf8')).toContain(
124+
'daemon_shutdown_platform_owner_failed',
125+
);
126+
expect(fs.existsSync(paths.infoPath)).toBe(true);
127+
expect(fs.existsSync(paths.lockPath)).toBe(true);
128+
expect(clearLock).not.toHaveBeenCalled();
129+
expect(pluginClose).toHaveBeenCalledOnce();
130+
expect(finalize).toHaveBeenCalledOnce();
131+
expect(await runtime.close()).toBe(report);
132+
expect(shutdown).toHaveBeenCalledOnce();
133+
});
134+
test('settles all owners and preserves the failure across concurrent and repeated shutdown', async () => {
135+
const failedRef = providerRuntimeOwner('limrun', 'failed');
136+
const healthyRef = providerRuntimeOwner('limrun', 'healthy');
137+
const failure = new Error('owner refused');
138+
const failedShutdown = vi.fn(async () => {
139+
throw failure;
140+
});
141+
const healthyShutdown = vi.fn(async () => {});
142+
const failed = providerRuntime({
143+
ref: failedRef,
144+
load: async () => ({ ...runtimeOwner({ ref: failedRef }), shutdown: failedShutdown }),
145+
});
146+
const healthy = providerRuntime({
147+
ref: healthyRef,
148+
load: async () => ({ ...runtimeOwner({ ref: healthyRef }), shutdown: healthyShutdown }),
149+
});
150+
const runtimeGateway = gateway([failed, healthy]);
151+
for (const ref of [failedRef, healthyRef]) {
152+
await runtimeGateway.bind({
153+
device,
154+
intent: { kind: 'exact-owner', owner: ref, fence: { token: 'fence', generation: 1 } },
155+
scope,
156+
});
157+
}
158+
159+
const first = runtimeGateway.shutdown();
160+
const second = runtimeGateway.shutdown();
161+
const results = await Promise.allSettled([first, second]);
162+
for (const result of results) {
163+
expect(result.status).toBe('rejected');
164+
if (result.status === 'rejected') {
165+
expect(result.reason).toBeInstanceOf(AggregateError);
166+
expect(result.reason.message).toContain('failed');
167+
expect(result.reason.errors[0].cause).toBe(failure);
168+
}
169+
}
170+
await expect(runtimeGateway.shutdown()).rejects.toBe(
171+
(results[0] as PromiseRejectedResult).reason,
172+
);
173+
expect(failedShutdown).toHaveBeenCalledOnce();
174+
expect(healthyShutdown).toHaveBeenCalledOnce();
175+
await expect(
176+
runtimeGateway.bind({
177+
device,
178+
intent: {
179+
kind: 'exact-owner',
180+
owner: healthyRef,
181+
fence: { token: 'fence', generation: 1 },
182+
},
183+
scope,
184+
}),
185+
).rejects.toThrow('shut down');
186+
});
31187
test('loads only the selected Apple owner for screenshot facts and binding', async () => {
32188
const appleDevice: DeviceInfo = {
33189
platform: 'apple',

‎src/platform-runtime-gateway.ts‎

Lines changed: 29 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ export function createComposedPlatformRuntimeGateway(options: {
6262
const localLoads = new Map<Platform, Promise<PlatformRuntimeOwner>>();
6363
const providerLoads = new Map<PlatformRuntimeProviderModule, Promise<PlatformRuntimeOwner>>();
6464
const loadedOwners = new Map<string, PlatformRuntimeOwner>();
65+
let closing: Promise<void> | undefined;
6566
let hostLoad: Promise<PlatformRuntimeHost> | undefined;
6667
const loadHost = async () => {
6768
hostLoad ??= options.loadHost();
@@ -105,6 +106,7 @@ export function createComposedPlatformRuntimeGateway(options: {
105106
return owner;
106107
};
107108
const loadLocal = async (family: Platform) => {
109+
if (closing) throw runtimeContractError('Platform runtime gateway is shut down');
108110
const existing = localLoads.get(family);
109111
if (existing) return await existing;
110112
const module = options.modules.get(family);
@@ -132,6 +134,7 @@ export function createComposedPlatformRuntimeGateway(options: {
132134
}
133135
};
134136
const loadProvider = async (module: PlatformRuntimeProviderModule) => {
137+
if (closing) throw runtimeContractError('Platform runtime gateway is shut down');
135138
const existing = providerLoads.get(module);
136139
if (existing) return await existing;
137140
const pending = loadHost().then(async (host) => {
@@ -184,14 +187,32 @@ export function createComposedPlatformRuntimeGateway(options: {
184187
}
185188
return await bindAndValidate(await loadLocal(request.device.platform), request);
186189
},
187-
shutdown: async () => {
188-
await Promise.allSettled(
189-
[...loadedOwners.values()].map(async (owner) => await owner.shutdown()),
190-
);
191-
loadedOwners.clear();
192-
localLoads.clear();
193-
providerLoads.clear();
194-
},
190+
shutdown: () =>
191+
(closing ??= Promise.resolve().then(async () => {
192+
await Promise.allSettled([...localLoads.values(), ...providerLoads.values()]);
193+
const owners = [...loadedOwners.entries()];
194+
const outcomes = await Promise.allSettled(
195+
owners.map(async ([, owner]) => owner.shutdown()),
196+
);
197+
loadedOwners.clear();
198+
localLoads.clear();
199+
providerLoads.clear();
200+
const failures = outcomes.flatMap((outcome, index) =>
201+
outcome.status === 'rejected'
202+
? [
203+
new Error(`Platform runtime owner ${owners[index]![0]} failed to shut down`, {
204+
cause: outcome.reason,
205+
}),
206+
]
207+
: [],
208+
);
209+
if (failures.length > 0) {
210+
throw new AggregateError(
211+
failures,
212+
`Platform runtime owner shutdown failed: ${failures.map((failure) => failure.message).join('; ')}`,
213+
);
214+
}
215+
})),
195216
});
196217
}
197218

‎src/sdk/host-types.ts‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,10 @@ export type AgentDeviceHost = Readonly<{
1818
client: AgentDeviceClient;
1919
/** Serializable attach config containing this instance's bearer token. Keep it inside the host. */
2020
clientConfig: AgentDeviceClientConfig;
21-
/** Reports unfinished teardown. Repeated calls share one close; never exits the process. */
21+
/**
22+
* Reports teardown outcomes. `platformOwners` is `closed`, `pending` when the shutdown budget
23+
* expires, or `failed` when an owner rejects. Pending or failed owners retain the daemon
24+
* registration until this process exits. Repeated calls share one close; never exits the process.
25+
*/
2226
close: () => Promise<ShutdownReport>;
2327
}>;

0 commit comments

Comments
 (0)