Skip to content

Commit a9edcb9

Browse files
committed
fix(limrun): recover when an instance ends under a lease
1 parent b42c7c2 commit a9edcb9

10 files changed

Lines changed: 228 additions & 18 deletions

File tree

‎packages/daemon/src/server/http/daemon-runtime.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -312,7 +312,8 @@ async function startDaemonRuntimeOnHost(
312312
token,
313313
sessionStore,
314314
leaseRegistry,
315-
endSessionsForLease: async (leaseId) => await endSessionsForLease(leaseId, 'LEASE_RELEASED'),
315+
endSessionsForLease: async (leaseId, reason) =>
316+
await endSessionsForLease(leaseId, reason ?? 'LEASE_RELEASED'),
316317
appLogAdmissionLedger,
317318
audioProbeAdmissionLedger,
318319
perfCaptureAdmissionLedger,

‎packages/daemon/src/server/http/http-server.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ const MAX_HTTP_RPC_BODY_BYTES = 1024 * 1024;
7373
export async function createDaemonHttpServer(options: {
7474
handleRequest: DaemonInvokeFn;
7575
leaseRegistry?: LeaseRegistry;
76-
endSessionsForLease?: (leaseId: string) => Promise<void>;
76+
endSessionsForLease?: (leaseId: string, reason?: 'PROVIDER_INSTANCE_ENDED') => Promise<void>;
7777
token?: string;
7878
retainArtifacts?: boolean;
7979
/** The runtime host's environment: read for the auth hook and handed to the download archive. */

‎packages/daemon/src/server/request-router.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -151,7 +151,7 @@ export type RequestRouterDeps = {
151151
leaseLifecycleProvider?: LeaseLifecycleProvider;
152152
cloudArtifactProvider?: CloudArtifactProvider;
153153
/** Ends the sessions opened under a lease the tenant just released, and clears their claims. */
154-
endSessionsForLease?: (leaseId: string) => Promise<void>;
154+
endSessionsForLease?: (leaseId: string, reason?: 'PROVIDER_INSTANCE_ENDED') => Promise<void>;
155155
providerAppCatalog?: ProviderAppCatalog;
156156
androidObservation: AndroidObservationAdapter;
157157
/** Builds the per-request command surface the device-facing runtimes execute through. */

‎packages/daemon/src/sessions/handlers/__tests__/lease.test.ts‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,98 @@ import {
2222
humanControlRequest,
2323
} from '../../../__tests__/support/human-control-fixtures.ts';
2424

25+
test('a provider-ended instance retires its lease and session before reporting its cause', async () => {
26+
const registry = new LeaseRegistry();
27+
const lease = registry.allocateLease({
28+
tenantId: 'tenant-a',
29+
runId: 'run-1',
30+
leaseProvider: 'limrun',
31+
leaseBackend: 'ios-instance',
32+
});
33+
const ended: Array<[string, string | undefined]> = [];
34+
const failure = new AppError('COMMAND_FAILED', 'The Limrun instance ended (InactivityTimeout).', {
35+
reason: 'provider-instance-ended',
36+
terminationReason: 'InactivityTimeout',
37+
});
38+
const run = () =>
39+
handleLeaseCommands({
40+
host: createTestHostEnvironment(),
41+
req: {
42+
token: 'test-token',
43+
session: 'default',
44+
command: 'lease_heartbeat',
45+
positionals: [],
46+
meta: {
47+
tenantId: lease.tenantId,
48+
runId: lease.runId,
49+
leaseId: lease.leaseId,
50+
leaseBackend: lease.backend,
51+
leaseProvider: lease.leaseProvider,
52+
},
53+
},
54+
sessionName: 'default',
55+
sessionStore: makeSessionStore('agent-device-ended-provider-'),
56+
leaseRegistry: registry,
57+
leaseLifecycleProvider: {
58+
heartbeat: async () => {
59+
throw failure;
60+
},
61+
release: async () => ({}),
62+
},
63+
endSessionsForLease: async (id, reason) => {
64+
ended.push([id, reason]);
65+
},
66+
});
67+
await assert.rejects(run(), (error) => error === failure);
68+
assert.deepEqual(ended, [[lease.leaseId, 'PROVIDER_INSTANCE_ENDED']]);
69+
assert.equal(registry.listActiveLeases().length, 0);
70+
await assert.rejects(run(), (error: AppError) => error.code === 'UNAUTHORIZED');
71+
});
72+
73+
test('a heartbeat error with the same message does not end a lease without the typed reason', async () => {
74+
const registry = new LeaseRegistry();
75+
const lease = registry.allocateLease({
76+
tenantId: 'tenant-a',
77+
runId: 'run-1',
78+
leaseBackend: 'ios-instance',
79+
leaseProvider: 'limrun',
80+
});
81+
const failure = new AppError('COMMAND_FAILED', 'The Limrun instance ended (InactivityTimeout).');
82+
const ended: string[] = [];
83+
await assert.rejects(
84+
handleLeaseCommands({
85+
host: createTestHostEnvironment(),
86+
req: {
87+
token: 'test-token',
88+
session: 'default',
89+
command: 'lease_heartbeat',
90+
positionals: [],
91+
meta: {
92+
tenantId: lease.tenantId,
93+
runId: lease.runId,
94+
leaseId: lease.leaseId,
95+
leaseBackend: lease.backend,
96+
leaseProvider: lease.leaseProvider,
97+
},
98+
},
99+
sessionName: 'default',
100+
sessionStore: makeSessionStore('agent-device-heartbeat-error-'),
101+
leaseRegistry: registry,
102+
leaseLifecycleProvider: {
103+
heartbeat: async () => {
104+
throw failure;
105+
},
106+
},
107+
endSessionsForLease: async (id) => {
108+
ended.push(id);
109+
},
110+
}),
111+
(error) => error === failure,
112+
);
113+
assert.deepEqual(ended, []);
114+
assert.equal(registry.listActiveLeases().length, 1);
115+
});
116+
25117
for (const operation of ['allocate', 'release'] as const) {
26118
test(`host activation drains provider lease ${operation} before reporting active`, async () => {
27119
const registry = new LeaseRegistry();

‎packages/daemon/src/sessions/handlers/lease.ts‎

Lines changed: 32 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ type LeaseHandlerArgs = {
5454
providerCredentials?: DaemonProviderCredentials;
5555
leaseLifecycleProvider?: LeaseLifecycleProvider;
5656
cloudArtifactProvider?: CloudArtifactProvider;
57-
endSessionsForLease?: (leaseId: string) => Promise<void>;
57+
endSessionsForLease?: (leaseId: string, reason?: 'PROVIDER_INSTANCE_ENDED') => Promise<void>;
5858
};
5959

6060
export async function handleLeaseCommands(args: LeaseHandlerArgs): Promise<DaemonResponse | null> {
@@ -151,17 +151,8 @@ export async function handleLeaseCommands(args: LeaseHandlerArgs): Promise<Daemo
151151
};
152152
});
153153
}
154-
case 'lease_heartbeat': {
155-
const lease = leaseRegistry.heartbeatLease(leaseScopeToHeartbeatRequest(leaseScope));
156-
const providerData = await leaseLifecycleProvider?.heartbeat?.(
157-
lease,
158-
leaseLifecycleContext(req, args.host),
159-
);
160-
return {
161-
ok: true,
162-
data: { lease, ...(providerData ? { provider: providerData } : {}) },
163-
};
164-
}
154+
case 'lease_heartbeat':
155+
return await heartbeatLease(args, leaseScope);
165156
case 'lease_release': {
166157
const releaseRequest = leaseScopeToReleaseRequest(leaseScope);
167158
const lease = leaseRegistry.getLease(releaseRequest);
@@ -313,6 +304,35 @@ async function releaseAllocationForGoneRequester(
313304
return canceledAllocationError(lease, outcome, releaseError);
314305
}
315306

307+
async function heartbeatLease(
308+
args: LeaseHandlerArgs,
309+
leaseScope: LeaseScope,
310+
): Promise<DaemonResponse> {
311+
const { leaseRegistry, leaseLifecycleProvider, req, endSessionsForLease } = args;
312+
const lease = leaseRegistry.heartbeatLease(leaseScopeToHeartbeatRequest(leaseScope));
313+
let providerData: Record<string, unknown> | undefined;
314+
let ended: AppError | undefined;
315+
try {
316+
providerData = await leaseLifecycleProvider?.heartbeat?.(
317+
lease,
318+
leaseLifecycleContext(req, args.host),
319+
);
320+
} catch (error) {
321+
if (!(error instanceof AppError) || error.details?.reason !== 'provider-instance-ended')
322+
throw error;
323+
await releaseLease(leaseRegistry, leaseLifecycleProvider, lease, leaseReleaseRequestFor(lease));
324+
ended = error;
325+
}
326+
if (ended) {
327+
await endSessionsForLease?.(lease.leaseId, 'PROVIDER_INSTANCE_ENDED');
328+
throw ended;
329+
}
330+
return {
331+
ok: true,
332+
data: { lease, ...(providerData ? { provider: providerData } : {}) },
333+
};
334+
}
335+
316336
function canceledAllocationError(
317337
lease: DeviceLease,
318338
outcome: LeaseReleaseOutcome,

‎packages/daemon/src/sessions/lease-lifecycle.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ export type ExpiredProviderLeaseRecovery = (lease: DeviceLease) => Promise<void>
2121

2222
export type SessionTeardown = (ref: SessionRef) => Promise<void>;
2323

24-
export type LeaseEndReason = 'LEASE_RELEASED' | 'LEASE_EXPIRED';
24+
export type LeaseEndReason = 'LEASE_RELEASED' | 'LEASE_EXPIRED' | 'PROVIDER_INSTANCE_ENDED';
2525

2626
export async function releaseExpiredProviderLease(
2727
recoverExpiredLease: ExpiredProviderLeaseRecovery | undefined,

‎packages/daemon/src/sessions/request-handler-chain.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ type RequestHandlerChainParams = {
4444
providerCredentials?: DaemonProviderCredentials;
4545
leaseLifecycleProvider?: LeaseLifecycleProvider;
4646
cloudArtifactProvider?: CloudArtifactProvider;
47-
endSessionsForLease?: (leaseId: string) => Promise<void>;
47+
endSessionsForLease?: (leaseId: string, reason?: 'PROVIDER_INSTANCE_ENDED') => Promise<void>;
4848
providerAppCatalog?: ProviderAppCatalog;
4949
invoke: DaemonInvokeFn;
5050
invokeReplayAction?: DaemonInvokeFn;

‎packages/provider-limrun/src/runtime-dependencies.test.ts‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ const state = vi.hoisted(() => ({
3434
]),
3535
tunnelClose: vi.fn(),
3636
disconnect: vi.fn(),
37+
connectionState: 'connected' as 'connected' | 'reconnecting' | 'disconnected',
3738
}));
3839

3940
vi.mock('@limrun/api', () => ({
@@ -57,6 +58,9 @@ vi.mock('@limrun/api', () => ({
5758
};
5859
}),
5960
list: vi.fn(),
61+
get: vi.fn(async () => ({
62+
status: { state: 'terminated', terminationReason: 'InactivityTimeout' },
63+
})),
6064
delete: vi.fn(async () => undefined),
6165
};
6266

@@ -70,9 +74,39 @@ vi.mock('@limrun/api', () => ({
7074
},
7175
}));
7276

77+
test('heartbeat reports a terminated instance with its typed cause', async () => {
78+
const runtime = createLimrunRuntime(
79+
{ apiKey: 'lim_test_key' },
80+
createContractFixture().dependencies,
81+
);
82+
const lease: DeviceLease = {
83+
leaseId: 'lease-android',
84+
tenantId: 'team-a',
85+
runId: 'run-a',
86+
backend: 'android-instance',
87+
leaseProvider: 'limrun',
88+
createdAt: 1,
89+
heartbeatAt: 1,
90+
expiresAt: 60_001,
91+
};
92+
try {
93+
await runtime.leaseLifecycle.allocate?.(lease);
94+
await assert.rejects(
95+
runtime.leaseLifecycle.heartbeat!(lease),
96+
(error: unknown) =>
97+
error instanceof AppError &&
98+
error.details?.reason === 'provider-instance-ended' &&
99+
error.details.terminationReason === 'InactivityTimeout',
100+
);
101+
} finally {
102+
await runtime.shutdown();
103+
}
104+
});
105+
73106
vi.mock('@limrun/api/instance-client', () => ({
74107
createInstanceClient: vi.fn(async () => ({
75108
disconnect: state.disconnect,
109+
getConnectionState: () => state.connectionState,
76110
tap: vi.fn(async () => {
77111
state.clientCalls.push('tap');
78112
}),
@@ -91,6 +125,43 @@ vi.mock('@limrun/api/instance-client', () => ({
91125
})),
92126
}));
93127

128+
test('an attached instance with a closed connection ends only after reconnecting stops', async () => {
129+
const runtime = createLimrunRuntime(
130+
{
131+
instances: {
132+
android: { apiUrl: 'https://android.example', token: 'token', adbUrl: 'wss://adb.example' },
133+
},
134+
},
135+
createContractFixture().dependencies,
136+
);
137+
const lease: DeviceLease = {
138+
leaseId: 'lease-android',
139+
tenantId: 'team-a',
140+
runId: 'run-a',
141+
backend: 'android-instance',
142+
leaseProvider: 'limrun',
143+
createdAt: 1,
144+
heartbeatAt: 1,
145+
expiresAt: 60_001,
146+
};
147+
try {
148+
await runtime.leaseLifecycle.allocate?.(lease);
149+
state.connectionState = 'reconnecting';
150+
assert.equal(await runtime.leaseLifecycle.heartbeat!(lease), undefined);
151+
state.connectionState = 'disconnected';
152+
await assert.rejects(
153+
runtime.leaseLifecycle.heartbeat!(lease),
154+
(error: unknown) =>
155+
error instanceof AppError &&
156+
error.details?.reason === 'provider-instance-ended' &&
157+
error.details.terminationReason === 'Unknown',
158+
);
159+
} finally {
160+
state.connectionState = 'connected';
161+
await runtime.shutdown();
162+
}
163+
});
164+
94165
test('factory uses the injected Android and host adapters as its construction seam', async () => {
95166
const fixture = createContractFixture();
96167
const runtime = createLimrunRuntime({ apiKey: 'lim_test_key' }, fixture.dependencies);

‎packages/provider-limrun/src/runtime.ts‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,7 @@ class LimrunRuntimeImplementation implements ProviderDeviceRuntime {
141141

142142
readonly leaseLifecycle: LeaseLifecycleProvider = {
143143
allocate: async (lease, context) => await this.allocate(lease, context),
144+
heartbeat: async (lease, context) => await this.heartbeat(lease, context),
144145
release: async (lease) => await this.release(lease),
145146
};
146147

@@ -741,6 +742,31 @@ class LimrunRuntimeImplementation implements ProviderDeviceRuntime {
741742
};
742743
}
743744

745+
private async heartbeat(
746+
lease: DeviceLease,
747+
context?: LeaseLifecycleContext,
748+
): Promise<Record<string, unknown> | undefined> {
749+
const session = this.liveSessionForLease(lease.leaseId);
750+
if (!session) return undefined;
751+
let terminationReason: string;
752+
if (this.limrun && session.ownership === 'created') {
753+
const instance =
754+
session.platform === 'ios'
755+
? await this.limrun.iosInstances.get(session.instanceId, { signal: context?.signal })
756+
: await this.limrun.androidInstances.get(session.instanceId, { signal: context?.signal });
757+
if (instance.status.state !== 'terminated') return undefined;
758+
terminationReason = instance.status.terminationReason ?? 'Unknown';
759+
} else {
760+
if (session.client.getConnectionState() !== 'disconnected') return undefined;
761+
terminationReason = 'Unknown';
762+
}
763+
throw new AppError('COMMAND_FAILED', `The Limrun instance ended (${terminationReason}).`, {
764+
reason: 'provider-instance-ended',
765+
terminationReason,
766+
instanceId: session.instanceId,
767+
});
768+
}
769+
744770
private async release(lease: DeviceLease): Promise<Record<string, unknown> | undefined> {
745771
// The take happens in this step, before anything is loaded: the release owns the lease from here, and no
746772
// await in between can let the allocation still waiting on its create answer ahead of it.

‎website/docs/docs/limrun.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ Limrun ends an instance after its inactivity timeout, which an idle session can
5656
Two timers can end an idle Limrun session.
5757

5858
- The session lease. Each command that uses the lease renews it for five minutes. After five minutes without such a command the lease expires, and the daemon ends the session. The next command that uses the lease allocates a new lease, and the app you opened is not restored.
59-
- The Limrun instance. An instance that agent-device creates ends after five minutes without activity, the same window as the lease. An instance you attach to keeps the inactivity timeout it was created with. When Limrun ends an instance, every later command fails with `WebSocket is not connected` until the lease expires or you run `disconnect`.
59+
- The Limrun instance. An instance that agent-device creates ends after five minutes without activity, the same window as the lease. An instance you attach to keeps the inactivity timeout it was created with. When a created instance ends, the next command reports why it ended. Run `open` again to start a new instance. For an attached instance, replace its connection details before you open again.
6060

6161
`LIMRUN_KEEP_ALIVE` keeps the instance running past five minutes. It does not renew the lease.
6262

0 commit comments

Comments
 (0)