Skip to content

Commit 385fd27

Browse files
committed
fix(limrun): observe every promise a disposal starts, in the step that starts it
A bounded wait now answers for both of a work item's outcomes: a capture record or a delete that refuses is an outcome in its own right, not a failure for this route to re-raise, and it never reaches the process as an unhandled rejection — inside the budget or after it. The two promises a disposal starts beside the instance delete are observed the moment they are started, so a session disconnect or a tunnel close that throws during a gated delete has its observer already beside it. The daemon treats one loose rejection as fatal, and a disposal has other people's sessions. A release whose delete Limrun confirmed still answers for the instance it ended: a refused capture record reports as a teardown warning naming the record, not as the release's failure. The disposal's outcome keeps its shape from the delete alone — no capture or local-teardown result can change what was confirmed gone. Tests: two `createLimrunRuntime` cases with a gated delete and a throwing disconnect, and with a throwing capture close, each with a process-level unhandledRejection spy. The alias case now drives the platform module's binding, through `resolveOpenTarget`, and asserts the name stops answering once the lease is released, so the exported runtime type keeps the surface it had before this PR.
1 parent 4ae7368 commit 385fd27

6 files changed

Lines changed: 373 additions & 85 deletions

File tree

Lines changed: 202 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,202 @@
1+
import assert from 'node:assert/strict';
2+
import { test, vi, expect } from 'vitest';
3+
import type { DeviceLease } from '@agent-device/contracts/device';
4+
import { createLimrunRuntime } from './runtime.ts';
5+
import { stubLimrunRuntimeDependencies } from './runtime-dependencies.fixtures.ts';
6+
import { mkdtempForTestSync } from './tmp-dir.fixtures.ts';
7+
import type { LimrunRuntimeDependencies } from './runtime-dependencies.ts';
8+
9+
/*
10+
* A disposal starts its capture end and its local teardown beside the instance delete and joins them only
11+
* after it. A step of either one that refuses inside that window has this disposal as its observer from the
12+
* moment it started: nothing a disposal begins may reach the process as an unhandled rejection, and a
13+
* refused capture or teardown never changes what the delete answered for.
14+
*/
15+
16+
const state = vi.hoisted(() => ({
17+
/** Instance deletes this runtime asked Limrun for; each waits for its case to answer it. */
18+
deletes: [] as Array<{ instanceId: string; report: () => void }>,
19+
disconnects: 0,
20+
/** Made to throw by the case whose session disconnect refuses while its delete is still owed. */
21+
disconnectThrows: false,
22+
captureCloses: 0,
23+
/** Made to throw by the case whose capture tunnel close refuses as the release writes its record. */
24+
captureCloseThrows: false,
25+
}));
26+
27+
vi.mock('@limrun/api', () => ({
28+
default: class MockLimrun {
29+
readonly iosInstances = { create: vi.fn(), list: vi.fn(), delete: vi.fn() };
30+
readonly androidInstances = {
31+
create: async () => ({
32+
metadata: { id: 'android-instance-1' },
33+
status: {
34+
token: 'instance-token',
35+
apiUrl: 'https://android.example',
36+
adbWebSocketUrl: 'wss://adb.example',
37+
},
38+
}),
39+
list: vi.fn(async () => ({ getPaginatedItems: () => [] })),
40+
// Every delete waits for its case to answer it, so a release can be held at its delete while the
41+
// local teardown it started beside has long since refused.
42+
delete: vi.fn(async (instanceId: string) => {
43+
let report!: () => void;
44+
const answered = new Promise<void>((answeredNow) => {
45+
report = answeredNow;
46+
});
47+
state.deletes.push({ instanceId, report });
48+
await answered;
49+
}),
50+
};
51+
readonly assets = { getOrUpload: vi.fn(), list: vi.fn(async () => []) };
52+
},
53+
}));
54+
55+
vi.mock('@limrun/api/instance-client', () => ({
56+
createInstanceClient: vi.fn(async () => ({
57+
disconnect: () => {
58+
state.disconnects += 1;
59+
if (state.disconnectThrows) throw new Error('instance client refused to disconnect');
60+
},
61+
keepAlive: () => {},
62+
openUrl: vi.fn(),
63+
sendAsset: vi.fn(),
64+
pressKey: vi.fn(),
65+
setText: vi.fn(),
66+
tap: vi.fn(),
67+
startAdbTunnel: vi.fn(async () => {
68+
throw new Error('no ADB tunnel is raised by these cases');
69+
}),
70+
startTunnel: vi.fn(async () => ({
71+
tunnelId: 't1',
72+
selectors: [],
73+
close: () => {
74+
state.captureCloses += 1;
75+
if (state.captureCloseThrows) throw new Error('capture tunnel refused to close');
76+
},
77+
})),
78+
getTunnelStatus: vi.fn(async () => ({})),
79+
})),
80+
}));
81+
82+
test('a local teardown that refuses beside a pending delete is reported, not loosed on the process', async () => {
83+
const rejections = watchUnhandledRejections();
84+
state.deletes.length = 0;
85+
state.disconnectThrows = true;
86+
const runtime = createLimrunRuntime({ apiKey: 'lim_test_key' }, testDependencies());
87+
const lease = androidLease('lease-throwing-disconnect');
88+
let reportDelete: (() => void) | undefined;
89+
try {
90+
await runtime.leaseLifecycle.allocate?.(lease);
91+
// The release holds at its delete while the session disconnect it started beside refuses.
92+
const released = runtime.leaseLifecycle.release?.(lease);
93+
await vi.waitFor(() => expect(state.deletes).toHaveLength(1));
94+
expect(state.disconnects).toBe(1);
95+
reportDelete = state.deletes[0]?.report;
96+
reportDelete?.();
97+
98+
// The delete's confirmation is the release's answer; what the local teardown came to rides beside it
99+
// as the warning it is: a host resource this process could not put down.
100+
const record = (await released) as Record<string, unknown>;
101+
expect(record.limrunInstanceId).toBe('android-instance-1');
102+
const warnings = warningsOf(record);
103+
expect(warnings.map((warning) => warning.code)).toEqual(['LIMRUN_TEARDOWN_INCOMPLETE']);
104+
expect(String(warnings[0]?.message)).toContain('Local teardown refused to finish');
105+
rejections.assertIsSilent();
106+
} finally {
107+
rejections.stop();
108+
reportDelete?.();
109+
state.deletes.forEach(({ report }) => report());
110+
state.deletes.length = 0;
111+
state.disconnectThrows = false;
112+
await runtime.shutdown();
113+
}
114+
});
115+
116+
test('a capture close that refuses still leaves the delete it ran beside answered for', async () => {
117+
const rejections = watchUnhandledRejections();
118+
const harDir = mkdtempForTestSync('limrun-har-observe-');
119+
state.deletes.length = 0;
120+
state.captureCloseThrows = true;
121+
const runtime = createLimrunRuntime(
122+
{ apiKey: 'lim_test_key', networkHarDir: harDir },
123+
testDependencies(),
124+
);
125+
const lease = androidLease('lease-throwing-capture-close');
126+
try {
127+
await runtime.leaseLifecycle.allocate?.(lease);
128+
await runtime.leaseLifecycle.allocate?.(lease, { flags: { providerNetworkLogs: true } });
129+
let reportDelete: (() => void) | undefined;
130+
try {
131+
const released = runtime.leaseLifecycle.release?.(lease);
132+
await vi.waitFor(() => expect(state.deletes).toHaveLength(1));
133+
reportDelete = state.deletes[0]?.report;
134+
reportDelete?.();
135+
136+
// The record's write ends with the tunnel's close, and that close refuses inside the disposal's
137+
// join window: the instance the delete confirmed is gone, and the release answers for it.
138+
const record = (await released) as Record<string, unknown>;
139+
expect(record.limrunInstanceId).toBe('android-instance-1');
140+
expect(state.captureCloses).toBe(1);
141+
expect(warningsOf(record).map((warning) => warning.message)).toEqual([
142+
'Capture record refused to be written',
143+
]);
144+
rejections.assertIsSilent();
145+
} finally {
146+
reportDelete?.();
147+
}
148+
} finally {
149+
rejections.stop();
150+
state.deletes.forEach(({ report }) => report());
151+
state.deletes.length = 0;
152+
state.captureCloseThrows = false;
153+
await runtime.shutdown();
154+
}
155+
});
156+
157+
function warningsOf(record: Record<string, unknown>): Array<Record<string, unknown>> {
158+
return Array.isArray(record.warnings) ? (record.warnings as Record<string, unknown>[]) : [];
159+
}
160+
161+
/**
162+
* Collects the rejections the process would die on: the daemon's failure mode for one is to take every
163+
* other session down with it, so a case whose disposal looses one fails here rather than in production.
164+
*/
165+
function watchUnhandledRejections(): {
166+
assertIsSilent: () => void;
167+
stop: () => void;
168+
} {
169+
const seen: unknown[] = [];
170+
const watch = (reason: unknown): void => {
171+
seen.push(reason);
172+
};
173+
process.on('unhandledRejection', watch);
174+
return {
175+
assertIsSilent: () => {
176+
if (seen.length === 0) return;
177+
assert.fail(`expected no unhandled rejection, got ${String(seen[0])}`);
178+
},
179+
stop: () => process.off('unhandledRejection', watch),
180+
};
181+
}
182+
183+
function androidLease(leaseId: string): DeviceLease {
184+
return {
185+
leaseId,
186+
tenantId: 'team-a',
187+
runId: 'run-a',
188+
backend: 'android-instance',
189+
leaseProvider: 'limrun',
190+
createdAt: 1,
191+
heartbeatAt: 1,
192+
expiresAt: 60_001,
193+
};
194+
}
195+
196+
/**
197+
* The host and platform adapters these cases run on: every one of them answers a fixed value. The client
198+
* connection and the instance delete are driven from the mock `state` above.
199+
*/
200+
function testDependencies(): LimrunRuntimeDependencies {
201+
return stubLimrunRuntimeDependencies();
202+
}

‎packages/provider-limrun/src/lease-release.ts‎

Lines changed: 54 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import type { DeviceLease } from '@agent-device/contracts/device';
33
import { AppError, errorMessage } from '@agent-device/kernel/errors';
44
import { cleanupLimrunAndroidAdbTunnel } from './android.ts';
55
import { LIMRUN_SINGLE_ATTEMPT } from './client-options.ts';
6+
import type { LimrunBudgetOutcome } from './lifecycle-budget.ts';
67
import { LIMRUN_PROVIDER, platformForLimrunLeaseBackend, type LimrunPlatform } from './device.ts';
78
import type {
89
LimrunDeleteResult,
@@ -67,8 +68,12 @@ export async function disposeLimrunLeaseOwner(
6768
host: LimrunDisposalHost,
6869
owner: LimrunLeaseOwner,
6970
): Promise<LimrunDisposalOutcome> {
70-
const { awaitLimrunWithinBudget, joinOutcomeFailures, workOutstandingFailure } =
71-
await import('./lifecycle-budget.ts');
71+
const {
72+
awaitLimrunWithinBudget,
73+
joinOutcomeFailures,
74+
keepLimrunObserved,
75+
workOutstandingFailure,
76+
} = await import('./lifecycle-budget.ts');
7277
const create = await awaitLimrunWithinBudget(owner.creation, CREATE_ANSWER_BUDGET_MS);
7378
// The create answered too late for this disposal to wait on it; what it goes on to name is still this
7479
// disposal's to end.
@@ -81,9 +86,11 @@ export async function disposeLimrunLeaseOwner(
8186
if (instanceId === undefined) return await disposeUnrented(host, owner);
8287
// The capture's record is optional reporting, and local teardown can outlive its own steps: neither is
8388
// allowed to stand between this disposal and the delete that stops the instance from being billed, so both
84-
// are started here and joined after it.
85-
const capture = host.endCapture(owner);
86-
const local = releaseOwnerResources(owner);
89+
// are started here and joined after it. Both are observed in this same step: a step of either one that
90+
// refuses before the join below is reached has this disposal as its observer from the moment it started,
91+
// so no refusal of a resource this route already owns can reach the process as an unhandled rejection.
92+
const capture = keepLimrunObserved(host.endCapture(owner));
93+
const local = keepLimrunObserved(releaseOwnerResources(owner));
8794
const instance = await endInstance(host, owner, session, instanceId);
8895
const stuck = await awaitLocalTeardown(local);
8996
// The record is optional reporting joined last: a capture that refuses to be written cannot keep a release
@@ -94,16 +101,28 @@ export async function disposeLimrunLeaseOwner(
94101
instanceId,
95102
failure: joinOutcomeFailures(
96103
instance.failure,
97-
joinOutcomeFailures(
98-
stuck,
99-
joined.ok ? undefined : workOutstandingFailure('Capture record', CAPTURE_RECORD_BUDGET_MS),
100-
),
104+
joinOutcomeFailures(stuck, captureRecordOutcome(joined, workOutstandingFailure)),
101105
),
102106
cause: instance.cause,
103107
capture: joined.ok ? joined.value : undefined,
104108
};
105109
}
106110

111+
/**
112+
* What the joined capture record leaves to report: nothing, a refusal of the record itself, or a wait
113+
* that ran out. A refusal is reported as what it is, because a record that will never be written is not
114+
* the same thing as one that may still arrive.
115+
*/
116+
function captureRecordOutcome(
117+
joined: LimrunBudgetOutcome<SettledNetworkCapture | undefined>,
118+
outstanding: (work: string, budgetMs: number) => string,
119+
): string | undefined {
120+
if (joined.ok) return undefined;
121+
return 'rejected' in joined
122+
? 'Capture record refused to be written'
123+
: outstanding('Capture record', CAPTURE_RECORD_BUDGET_MS);
124+
}
125+
107126
/**
108127
* Ends what a disposal holds when the create it waited for never answered. Nothing is known to be running, so
109128
* the lease is kept held and refused rather than given back — and the create is watched, because whatever it
@@ -113,8 +132,11 @@ async function disposeUnansweredCreate(
113132
host: LimrunDisposalHost,
114133
owner: LimrunLeaseOwner,
115134
): Promise<LimrunDisposalOutcome> {
116-
const { joinOutcomeFailures, workOutstandingFailure } = await import('./lifecycle-budget.ts');
117-
const capture = awaitEndedCapture(host, owner);
135+
const { joinOutcomeFailures, keepLimrunObserved, workOutstandingFailure } =
136+
await import('./lifecycle-budget.ts');
137+
// Observed the step it starts in, like the disposal's other early work: this route answers only after
138+
// the local teardown below, so a capture end that refuses meanwhile has its observer already beside it.
139+
const capture = keepLimrunObserved(awaitEndedCapture(host, owner));
118140
const outstanding = workOutstandingFailure(
119141
`Instance create for lease '${owner.lease.leaseId}'`,
120142
CREATE_ANSWER_BUDGET_MS,
@@ -240,15 +262,21 @@ async function endInstance(
240262
}
241263
const deleted = await awaitLimrunWithinBudget(requested, DELETE_ANSWER_BUDGET_MS);
242264
if (!deleted.ok) {
243-
void requested.then((answer) => {
244-
if (answer.failure === undefined) confirmLateDelete(owner, instanceId);
245-
});
265+
// Both branches of the request are answered for here: this continuation outlives its budget, and the
266+
// route that started it has stopped waiting, so nothing it reports later may reach the process as an
267+
// unhandled rejection.
268+
void requested.then(
269+
(answer) => {
270+
if (answer.failure === undefined) confirmLateDelete(owner, instanceId);
271+
},
272+
() => undefined,
273+
);
246274
return {
247275
cleanup: 'uncertain',
248-
failure: workOutstandingFailure(
249-
`Delete of instance '${instanceId}'`,
250-
DELETE_ANSWER_BUDGET_MS,
251-
),
276+
failure:
277+
'rejected' in deleted
278+
? `Delete of instance '${instanceId}' could not be attempted`
279+
: workOutstandingFailure(`Delete of instance '${instanceId}'`, DELETE_ANSWER_BUDGET_MS),
252280
cause: undefined,
253281
};
254282
}
@@ -303,21 +331,17 @@ async function sessionTheCreateBuilt(
303331
}
304332

305333
/**
306-
* Reports the local teardown a disposal had to stop waiting for. A step that refused is reported as one that
307-
* never answered: neither is something this runtime can still fix, and the delete that ran beside it has
308-
* already been answered.
334+
* Reports the local teardown a disposal had to stop waiting for. A step that refused and a step that never
335+
* answered are both reported as what they are: neither is something this runtime can still fix, and the
336+
* delete that ran beside it has already been answered.
309337
*/
310338
async function awaitLocalTeardown(local: Promise<void>): Promise<string | undefined> {
311339
const { awaitLimrunWithinBudget, workOutstandingFailure } = await import('./lifecycle-budget.ts');
312-
const stuck = workOutstandingFailure('Local teardown', LOCAL_TEARDOWN_BUDGET_MS);
313-
const ended = await awaitLimrunWithinBudget(
314-
local.then(
315-
(): string | undefined => undefined,
316-
(): string => stuck,
317-
),
318-
LOCAL_TEARDOWN_BUDGET_MS,
319-
);
320-
return ended.ok ? ended.value : stuck;
340+
const ended = await awaitLimrunWithinBudget(local, LOCAL_TEARDOWN_BUDGET_MS);
341+
if (ended.ok) return undefined;
342+
return 'rejected' in ended
343+
? 'Local teardown refused to finish'
344+
: workOutstandingFailure('Local teardown', LOCAL_TEARDOWN_BUDGET_MS);
321345
}
322346

323347
/** What a release answers for one lease: the instance it ended, with what it left and wrote. */

0 commit comments

Comments
 (0)