Skip to content

Commit ea3dbea

Browse files
committed
fix(daemon): retain Apple runners for leases kept on close
1 parent a2c169f commit ea3dbea

16 files changed

Lines changed: 605 additions & 54 deletions

‎packages/daemon/src/core/__tests__/session-store-lifetime.test.ts‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,25 @@ test('address reuse with the same record still starts a different lifetime', ()
7878
assert.equal(store.getRuntimeHints(ADDRESS), undefined);
7979
});
8080

81+
test('retained execution-host ownership follows the current session lifetime', () => {
82+
const store = makeSessionStore();
83+
const session = makeSession('default', {
84+
lease: { leaseId: 'lease-1', tenantId: 'tenant-a', runId: 'run-1' },
85+
});
86+
const old = store.publish(ADDRESS, session);
87+
store.retire(old, { retainExecutionHost: true });
88+
const [retained] = store.listRetainedExecutionHosts('lease-1');
89+
assert.ok(retained);
90+
assert.equal(store.ownsRetainedExecutionHost(retained), true);
91+
const successor = store.publish(ADDRESS, session);
92+
assert.equal(store.ownsRetainedExecutionHost(retained), false);
93+
assert.equal(store.retire(old, { retainExecutionHost: true }), false);
94+
assert.deepEqual(store.listRetainedExecutionHosts('lease-1'), []);
95+
store.retire(successor, { retainExecutionHost: true });
96+
assert.equal(store.releaseRetainedExecutionHost(retained), false);
97+
assert.equal(store.listRetainedExecutionHosts('lease-1').length, 1);
98+
});
99+
81100
test('retired updates cannot run their derivation or resurrect a record', () => {
82101
const store = makeSessionStore();
83102
const ref = store.publish(ADDRESS, makeSession('default'));

‎packages/daemon/src/core/platform-resource-cleanup.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ export type PlatformResourceCleanup = Readonly<{
1616
device: DeviceInfo;
1717
shutdownRequested: boolean;
1818
hasScreenRecording: boolean;
19-
hasLease: boolean;
19+
/** True when the lease is not kept on close, or when its close decision could not be inspected. */
20+
leaseWillEnd: boolean;
2021
}): boolean;
2122
}>;

‎packages/daemon/src/core/session-store.ts‎

Lines changed: 36 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,9 +49,15 @@ import {
4949
const REPAIR_TOMBSTONE_TTL_MS = 60 * 60_000;
5050
type SessionEntry = { current: SessionState };
5151
type SessionPatch = Partial<SessionState> | ((current: SessionState) => Partial<SessionState>);
52+
export type RetainedExecutionHost = Readonly<{
53+
address: string;
54+
device: SessionState['device'];
55+
lease: NonNullable<SessionState['lease']>;
56+
}>;
5257

5358
export class SessionStore {
5459
private readonly sessions = new Map<string, SessionEntry>();
60+
private readonly retainedExecutionHosts = new Map<string, RetainedExecutionHost>();
5561
private acceptingSessions = true;
5662
private readonly runtimeHints = new Map<string, SessionRuntimeHints>();
5763
private readonly sessionsDir: string;
@@ -105,6 +111,11 @@ export class SessionStore {
105111
this.assertPublishable(address);
106112
const entry = { current: session };
107113
this.sessions.set(address, entry);
114+
for (const retained of this.retainedExecutionHosts.values()) {
115+
if (devicesOverlap(retained.device.id, session.device.id)) {
116+
this.releaseRetainedExecutionHost(retained);
117+
}
118+
}
108119
this.clearIdleExpiryTombstone(address);
109120
return this.captureRef(address, entry);
110121
}
@@ -141,12 +152,35 @@ export class SessionStore {
141152
return next;
142153
}
143154

144-
retire(ref: SessionRef): boolean {
145-
if (!this.resolveCurrent(ref)) return false;
155+
retire(ref: SessionRef, options: { retainExecutionHost?: boolean } = {}): boolean {
156+
const session = this.resolveCurrent(ref);
157+
if (!session) return false;
158+
if (options.retainExecutionHost && session.lease) {
159+
this.retainedExecutionHosts.set(session.device.id, {
160+
address: ref.address,
161+
device: session.device,
162+
lease: { ...session.lease },
163+
});
164+
}
146165
this.runtimeHints.delete(ref.address);
147166
return this.sessions.delete(ref.address);
148167
}
149168

169+
listRetainedExecutionHosts(leaseId: string): RetainedExecutionHost[] {
170+
return Array.from(this.retainedExecutionHosts.values()).filter(
171+
(host) => host.lease.leaseId === leaseId,
172+
);
173+
}
174+
175+
ownsRetainedExecutionHost(host: RetainedExecutionHost): boolean {
176+
return this.retainedExecutionHosts.get(host.device.id) === host;
177+
}
178+
179+
releaseRetainedExecutionHost(host: RetainedExecutionHost): boolean {
180+
if (!this.ownsRetainedExecutionHost(host)) return false;
181+
return this.retainedExecutionHosts.delete(host.device.id);
182+
}
183+
150184
private captureRef(address: string, entry: SessionEntry): SessionRef {
151185
return Object.freeze({ address, session: entry.current, lifetime: entry });
152186
}

‎packages/daemon/src/server/__tests__/request-execution-scope-lease-session-end.test.ts‎

Lines changed: 255 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import { acquireDeviceClaim } from '@agent-device/daemon-contracts/device-claims
1616
import { existingSessionExecutionLockKeys } from '../request-binding.ts';
1717
import { withRequestExecutionLockKeys } from '../request-execution-locks.ts';
1818
import type { SessionStore } from '../../core/session-store.ts';
19+
import type { PlatformResourceCleanup } from '../../core/platform-resource-cleanup.ts';
1920
import type { SessionTeardown } from '../../sessions/lease-lifecycle.ts';
2021
import { createLeaseSessionEnd } from '../request-execution-scope.ts';
2122

@@ -67,7 +68,13 @@ function holdSessionLocks(
6768
return { finish, held };
6869
}
6970

70-
function leaseSessionEnd(sessionStore: SessionStore, teardownSession: SessionTeardown) {
71+
function leaseSessionEnd(
72+
sessionStore: SessionStore,
73+
teardownSession: SessionTeardown,
74+
cleanupSessionlessExecutionHost: PlatformResourceCleanup['cleanupSessionlessExecutionHost'] = vi.fn(
75+
async () => {},
76+
),
77+
) {
7178
const executionLocks = new Map<string, Promise<unknown>>();
7279
return {
7380
executionLocks,
@@ -76,12 +83,259 @@ function leaseSessionEnd(sessionStore: SessionStore, teardownSession: SessionTea
7683
sessionStore,
7784
executionLocks,
7885
teardownSession,
86+
cleanupSessionlessExecutionHost,
7987
}),
8088
};
8189
}
8290

8391
const settle = async (): Promise<void> => await new Promise((resolve) => setImmediate(resolve));
8492

93+
function retainExecutionHost(sessionStore: SessionStore, name = 'leased', leaseId = 'lease-1') {
94+
const ref = sessionStore.publish(
95+
name,
96+
makeIosSession(name, {
97+
lease: { leaseId, tenantId: 'tenant-a', runId: 'run-1' },
98+
}),
99+
);
100+
sessionStore.retire(ref, { retainExecutionHost: true });
101+
return ref;
102+
}
103+
104+
test.each(['LEASE_RELEASED', 'LEASE_EXPIRED'] as const)(
105+
'%s stops a retained host after its session has retired',
106+
async (reason) => {
107+
const sessionStore = makeSessionStore();
108+
const ref = retainExecutionHost(sessionStore);
109+
const cleanup = vi.fn(async () => {});
110+
const teardown = vi.fn(async () => {});
111+
const { ender } = leaseSessionEnd(sessionStore, teardown, cleanup);
112+
113+
await ender.end('lease-1', reason);
114+
115+
expect(teardown).not.toHaveBeenCalled();
116+
expect(cleanup).toHaveBeenCalledExactlyOnceWith(ref.session.device);
117+
expect(sessionStore.listRetainedExecutionHosts('lease-1')).toEqual([]);
118+
},
119+
);
120+
121+
test('expiry waiting behind close cleans the host that close retains before retiring', async () => {
122+
const sessionStore = makeSessionStore();
123+
const ref = sessionStore.publish(
124+
'leased',
125+
makeIosSession('leased', {
126+
lease: { leaseId: 'lease-1', tenantId: 'tenant-a', runId: 'run-1' },
127+
}),
128+
);
129+
const cleanup = vi.fn(async () => {});
130+
const teardown = vi.fn(async () => {});
131+
const { executionLocks, ender } = leaseSessionEnd(sessionStore, teardown, cleanup);
132+
const close = holdSessionLocks(executionLocks, sessionStore, 'leased');
133+
const ending = ender.end('lease-1', 'LEASE_EXPIRED');
134+
await settle();
135+
expect(cleanup).not.toHaveBeenCalled();
136+
137+
sessionStore.retire(ref, { retainExecutionHost: true });
138+
close.finish();
139+
await close.held;
140+
await ending;
141+
142+
expect(teardown).not.toHaveBeenCalled();
143+
expect(cleanup).toHaveBeenCalledExactlyOnceWith(ref.session.device);
144+
expect(sessionStore.listRetainedExecutionHosts('lease-1')).toEqual([]);
145+
});
146+
147+
test.each([false, true])(
148+
'lease end preserves a replacement owner with retained=%s',
149+
async (retained) => {
150+
const sessionStore = makeSessionStore();
151+
const old = retainExecutionHost(sessionStore);
152+
const cleanup = vi.fn(async () => {});
153+
const { executionLocks, ender } = leaseSessionEnd(sessionStore, async () => {}, cleanup);
154+
let unlock!: () => void;
155+
const held = withRequestExecutionLockKeys(
156+
executionLocks,
157+
existingSessionExecutionLockKeys(old.address, old.session.device.id),
158+
async () =>
159+
await new Promise<void>((resolve) => {
160+
unlock = resolve;
161+
}),
162+
);
163+
await settle();
164+
const ending = ender.end('lease-1', 'LEASE_RELEASED');
165+
await settle();
166+
const replacement = sessionStore.publish(
167+
'replacement',
168+
makeIosSession('replacement', {
169+
lease: { leaseId: 'lease-2', tenantId: 'tenant-b', runId: 'run-2' },
170+
}),
171+
);
172+
if (retained) sessionStore.retire(replacement, { retainExecutionHost: true });
173+
unlock();
174+
await held;
175+
await ending;
176+
177+
expect(cleanup).not.toHaveBeenCalled();
178+
expect(sessionStore.listRetainedExecutionHosts('lease-1')).toEqual([]);
179+
if (retained) expect(sessionStore.listRetainedExecutionHosts('lease-2')).toHaveLength(1);
180+
else expect(sessionStore.resolveCurrent(replacement)).toBeDefined();
181+
},
182+
);
183+
184+
test('release ends a same-lease reopen published at a new address while cleanup waits for its device', async () => {
185+
const sessionStore = makeSessionStore();
186+
const old = retainExecutionHost(sessionStore);
187+
const cleanup = vi.fn(async () => {});
188+
const teardown = vi.fn(async () => {});
189+
const { executionLocks, ender } = leaseSessionEnd(sessionStore, teardown, cleanup);
190+
let finishNativeOpen!: () => void;
191+
const nativeOpen = new Promise<void>((resolve) => {
192+
finishNativeOpen = resolve;
193+
});
194+
const reopening = withRequestExecutionLockKeys(
195+
executionLocks,
196+
existingSessionExecutionLockKeys('reopened', old.session.device.id),
197+
async () => {
198+
await nativeOpen;
199+
sessionStore.publish('reopened', makeIosSession('reopened', { lease: old.session.lease }));
200+
},
201+
);
202+
await settle();
203+
const ending = ender.end('lease-1', 'LEASE_RELEASED');
204+
await settle();
205+
expect(teardown).not.toHaveBeenCalled();
206+
finishNativeOpen();
207+
await reopening;
208+
await ending;
209+
210+
expect(cleanup).not.toHaveBeenCalled();
211+
expect(teardown).toHaveBeenCalledExactlyOnceWith(
212+
expect.objectContaining({ address: 'reopened' }),
213+
);
214+
expect(sessionStore.get('reopened')).toBeUndefined();
215+
expect(sessionStore.listRetainedExecutionHosts('lease-1')).toEqual([]);
216+
});
217+
218+
test('failed retained-host cleanup stays owned and throttles retries without another release request', async () => {
219+
const now = vi.spyOn(Date, 'now').mockReturnValue(0);
220+
try {
221+
const sessionStore = makeSessionStore();
222+
retainExecutionHost(sessionStore);
223+
const cleanup = vi
224+
.fn(async () => {})
225+
.mockRejectedValueOnce(new Error('device unreachable'))
226+
.mockRejectedValueOnce(new Error('device still unreachable'));
227+
const { ender } = leaseSessionEnd(sessionStore, async () => {}, cleanup);
228+
229+
await ender.end('lease-1', 'LEASE_RELEASED');
230+
expect(sessionStore.listRetainedExecutionHosts('lease-1')).toHaveLength(1);
231+
for (let elapsed = 1_000; elapsed < 10_000; elapsed += 1_000) {
232+
now.mockReturnValue(elapsed);
233+
await ender.retryPending();
234+
expect(cleanup).toHaveBeenCalledTimes(1);
235+
}
236+
now.mockReturnValue(10_000);
237+
await ender.retryPending();
238+
expect(cleanup).toHaveBeenCalledTimes(2);
239+
expect(sessionStore.listRetainedExecutionHosts('lease-1')).toHaveLength(1);
240+
now.mockReturnValue(19_999);
241+
await ender.retryPending();
242+
expect(cleanup).toHaveBeenCalledTimes(2);
243+
now.mockReturnValue(20_000);
244+
await ender.retryPending();
245+
expect(cleanup).toHaveBeenCalledTimes(3);
246+
expect(sessionStore.listRetainedExecutionHosts('lease-1')).toEqual([]);
247+
now.mockReturnValue(30_000);
248+
await ender.retryPending();
249+
expect(cleanup).toHaveBeenCalledTimes(3);
250+
} finally {
251+
now.mockRestore();
252+
}
253+
});
254+
255+
test('cancel preserves a retained host queued for cleanup for daemon shutdown', async () => {
256+
const sessionStore = makeSessionStore();
257+
const ref = retainExecutionHost(sessionStore);
258+
const cleanup = vi.fn(async () => {});
259+
const { executionLocks, ender } = leaseSessionEnd(sessionStore, async () => {}, cleanup);
260+
let unlock!: () => void;
261+
const held = withRequestExecutionLockKeys(
262+
executionLocks,
263+
existingSessionExecutionLockKeys(ref.address, ref.session.device.id),
264+
async () =>
265+
await new Promise<void>((resolve) => {
266+
unlock = resolve;
267+
}),
268+
);
269+
await settle();
270+
const ending = ender.end('lease-1', 'LEASE_EXPIRED');
271+
await settle();
272+
ender.cancel();
273+
unlock();
274+
await held;
275+
await ending;
276+
await ender.drain();
277+
await ender.retryPending();
278+
279+
expect(cleanup).not.toHaveBeenCalled();
280+
expect(sessionStore.listRetainedExecutionHosts('lease-1')).toHaveLength(1);
281+
});
282+
283+
test('retry does not queue duplicate cleanup and shutdown drains a running retained-host stop', async () => {
284+
const sessionStore = makeSessionStore();
285+
retainExecutionHost(sessionStore);
286+
let finish!: () => void;
287+
const cleanup = vi.fn(
288+
async () =>
289+
await new Promise<void>((resolve) => {
290+
finish = resolve;
291+
}),
292+
);
293+
const { ender } = leaseSessionEnd(sessionStore, async () => {}, cleanup);
294+
const ending = ender.end('lease-1', 'LEASE_EXPIRED');
295+
await settle();
296+
await ender.retryPending();
297+
expect(cleanup).toHaveBeenCalledTimes(1);
298+
ender.cancel();
299+
let drained = false;
300+
const drain = ender.drain().then(() => {
301+
drained = true;
302+
});
303+
await settle();
304+
expect(drained).toBe(false);
305+
finish();
306+
await ending;
307+
await drain;
308+
expect(sessionStore.listRetainedExecutionHosts('lease-1')).toEqual([]);
309+
});
310+
311+
test('a request holding the device lock can end its lease ahead of a queued background end', async () => {
312+
const sessionStore = makeSessionStore();
313+
const ref = retainExecutionHost(sessionStore);
314+
const cleanup = vi.fn(async () => {});
315+
const { executionLocks, ender } = leaseSessionEnd(sessionStore, async () => {}, cleanup);
316+
let release!: () => void;
317+
const continueRequest = new Promise<void>((resolve) => {
318+
release = resolve;
319+
});
320+
const request = withRequestExecutionLockKeys(
321+
executionLocks,
322+
existingSessionExecutionLockKeys(ref.address, ref.session.device.id),
323+
async () => {
324+
await continueRequest;
325+
await ender.end('lease-1', 'LEASE_RELEASED');
326+
},
327+
);
328+
await settle();
329+
const expiry = ender.end('lease-1', 'LEASE_EXPIRED');
330+
await settle();
331+
release();
332+
await request;
333+
await expiry;
334+
335+
expect(cleanup).toHaveBeenCalledTimes(1);
336+
expect(sessionStore.listRetainedExecutionHosts('lease-1')).toEqual([]);
337+
});
338+
85339
test('end publishes the recorded script of the session it ends', async () => {
86340
const sessionStore = makeSessionStore('agent-device-lease-end-script-');
87341
await publishLeasedSession(sessionStore, 'leased', 'lease-1');

0 commit comments

Comments
 (0)