Skip to content

Commit 483baac

Browse files
committed
fix(webdriver): give cloud session creation its own budget and stop leaking billed sessions
Cloud lease allocation ran under the generic 30s/1-retry request policy, so BrowserStack iOS real-device session creation (45-90s) aborted client-side at ~60s on most runs. Each timed-out POST /session still completed server-side and, being non-idempotent, was retried — leaving two billed provider sessions per failed open with no id to release them. - POST /session is its own phase: a 180s create budget (default), zero retries, and no request-bound abort, so the daemon always learns the session id. - lease_allocate carries a 300s allocation budget surfaced to providers as LeaseLifecycleContext.deadline, and a matching 330s client envelope that preserves the daemon on timeout (a reset would SIGKILL mid-create and orphan every billed session the daemon held). - The request's cancellation signal is ownership evidence: a session that completes after the requester left is released, not registered; a create that the transport gives up on surfaces typed evidence (provider + lease) so an operator can find and stop the maybe-orphaned session. Closes #1774
1 parent 2b6d04a commit 483baac

15 files changed

Lines changed: 694 additions & 49 deletions

File tree

‎packages/contracts/src/device-provider.ts‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,21 @@ export type DeviceLease = {
2525
export type LeaseLifecycleContext = {
2626
flags?: Readonly<Record<string, unknown>>;
2727
cwd?: string;
28+
/**
29+
* Request-bound cancellation: aborted once the requester is gone (explicit
30+
* cancel or client disconnect). For allocation it is an OWNERSHIP signal, not
31+
* an interrupt — a provider whose remote allocation has already committed
32+
* finishes it and releases the result rather than abandoning a billed
33+
* resource nobody holds the id of.
34+
*/
35+
signal?: AbortSignal;
36+
/**
37+
* Epoch-ms deadline by which `allocate` must have settled. The daemon derives
38+
* it from the same budget as the client's `lease_allocate` request envelope,
39+
* so a provider that fits its remote phases within it is never abandoned by
40+
* a client that stopped waiting first.
41+
*/
42+
deadline?: number;
2843
};
2944

3045
export type LeaseLifecycleProvider = {

‎packages/provider-webdriver/src/runtime-session.test.ts‎

Lines changed: 168 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import assert from 'node:assert/strict';
22
import { test } from 'vitest';
33
import type { DeviceLease } from '@agent-device/contracts/device';
4+
import { deviceFieldsFromPublicPlatform, type DeviceInfo } from '@agent-device/kernel/device';
45
import { AppError } from '@agent-device/kernel/errors';
56
import { createCloudWebDriverRuntime } from './runtime.ts';
67

@@ -47,6 +48,173 @@ test('session allocation preserves its primary failure when provider cleanup als
4748
}
4849
});
4950

51+
// #1774: a request already gone before the create is issued must create
52+
// NOTHING. Prepared provider resources are cleaned up and no `POST /session`
53+
// goes out, so there is no billed device session to leak.
54+
test('allocation canceled before create issues no session and cleans up prepared work', async () => {
55+
const previousFetch = globalThis.fetch;
56+
let sessionRequests = 0;
57+
let cleanupCalled = false;
58+
globalThis.fetch = async (input) => {
59+
if (String(input instanceof Request ? input.url : input).endsWith('/session')) {
60+
sessionRequests += 1;
61+
}
62+
return new Response(JSON.stringify({ value: { sessionId: 'wd-1', capabilities: {} } }), {
63+
status: 200,
64+
headers: { 'Content-Type': 'application/json' },
65+
});
66+
};
67+
const runtime = createCloudWebDriverRuntime({
68+
clientVersion: 'test',
69+
provider: 'webdriver-test',
70+
endpoint: 'https://webdriver.test/wd/hub/',
71+
platform: 'android',
72+
deviceName: 'Test device',
73+
requestPolicy: { retryAttempts: 0 },
74+
prepareSession: async ({ base }) => ({
75+
...base,
76+
cleanup: async () => {
77+
cleanupCalled = true;
78+
return undefined;
79+
},
80+
}),
81+
});
82+
const controller = new AbortController();
83+
controller.abort();
84+
85+
try {
86+
const allocate = runtime.leaseLifecycle.allocate;
87+
assert.ok(allocate);
88+
await assert.rejects(
89+
() => allocate(makeLease(), { signal: controller.signal }),
90+
(error: unknown) => {
91+
assert.ok(error instanceof AppError);
92+
assert.equal(error.details?.reason, 'request_canceled');
93+
return true;
94+
},
95+
);
96+
assert.equal(sessionRequests, 0, 'no POST /session may be issued once canceled');
97+
assert.equal(cleanupCalled, true);
98+
} finally {
99+
await runtime.shutdown();
100+
globalThis.fetch = previousFetch;
101+
}
102+
});
103+
104+
// #1774: the classic leak. The requester vanishes WHILE the provider is
105+
// allocating; the create still completes server-side. The manager must not
106+
// register that session (nobody is waiting on it) — it deletes it, holding the
107+
// id it just learned, so the billed session is released instead of orphaned.
108+
test('a session that completes after cancellation is released, not registered', async () => {
109+
const previousFetch = globalThis.fetch;
110+
const controller = new AbortController();
111+
let deletedSessionId: string | undefined;
112+
globalThis.fetch = async (input, init) => {
113+
const url = String(input instanceof Request ? input.url : input);
114+
const method = init?.method ?? 'GET';
115+
if (url.endsWith('/session') && method === 'POST') {
116+
// The client disconnects mid-create; the provider finishes anyway.
117+
controller.abort();
118+
return new Response(JSON.stringify({ value: { sessionId: 'wd-live', capabilities: {} } }), {
119+
status: 200,
120+
headers: { 'Content-Type': 'application/json' },
121+
});
122+
}
123+
if (url.endsWith('/session/wd-live') && method === 'DELETE') {
124+
deletedSessionId = 'wd-live';
125+
return new Response(JSON.stringify({ value: null }), {
126+
status: 200,
127+
headers: { 'Content-Type': 'application/json' },
128+
});
129+
}
130+
throw new Error(`unexpected ${method} ${url}`);
131+
};
132+
const runtime = createCloudWebDriverRuntime({
133+
clientVersion: 'test',
134+
provider: 'webdriver-test',
135+
endpoint: 'https://webdriver.test/wd/hub/',
136+
platform: 'android',
137+
deviceName: 'Test device',
138+
requestPolicy: { retryAttempts: 0 },
139+
});
140+
const lease = makeLease();
141+
142+
try {
143+
const allocate = runtime.leaseLifecycle.allocate;
144+
assert.ok(allocate);
145+
await assert.rejects(
146+
() => allocate(lease, { signal: controller.signal }),
147+
(error: unknown) => {
148+
assert.ok(error instanceof AppError);
149+
assert.equal(error.details?.reason, 'request_canceled');
150+
assert.equal(error.details?.releasedWebDriverSessionId, 'wd-live');
151+
return true;
152+
},
153+
);
154+
assert.equal(deletedSessionId, 'wd-live', 'the completed session must be deleted');
155+
// Nothing registered: the device is not owned and no session answers for it.
156+
assert.equal(runtime.getInteractor(makeDevice(lease)), undefined);
157+
} finally {
158+
await runtime.shutdown();
159+
globalThis.fetch = previousFetch;
160+
}
161+
});
162+
163+
// #1774: when the transport gives up on `POST /session`, the provider may still
164+
// finish it. The error names the lease the capabilities were labelled with so
165+
// an operator can find and stop the maybe-orphaned billed session, rather than
166+
// this process guessing at REST cleanup of a session it never owned.
167+
test('a create-timeout surfaces provider evidence for the maybe-leaked session', async () => {
168+
const previousFetch = globalThis.fetch;
169+
globalThis.fetch = async (input, init) => {
170+
if (String(input instanceof Request ? input.url : input).endsWith('/session')) {
171+
return await new Promise<Response>((_resolve, reject) => {
172+
init?.signal?.addEventListener('abort', () => reject(init.signal?.reason as Error));
173+
});
174+
}
175+
throw new Error('unexpected request');
176+
};
177+
const runtime = createCloudWebDriverRuntime({
178+
clientVersion: 'test',
179+
provider: 'browserstack',
180+
endpoint: 'https://webdriver.test/wd/hub/',
181+
platform: 'ios',
182+
deviceName: 'iPhone 17',
183+
requestPolicy: { retryAttempts: 0, sessionCreateTimeoutMs: 40 },
184+
});
185+
const lease = { ...makeLease(), leaseProvider: 'browserstack' };
186+
187+
try {
188+
const allocate = runtime.leaseLifecycle.allocate;
189+
assert.ok(allocate);
190+
await assert.rejects(
191+
() => allocate(lease),
192+
(error: unknown) => {
193+
assert.ok(error instanceof AppError);
194+
assert.equal(error.details?.reason, 'provider_session_create_timeout');
195+
assert.equal(error.details?.provider, 'browserstack');
196+
assert.equal(error.details?.leaseId, lease.leaseId);
197+
assert.match(String(error.details?.hint), new RegExp(lease.leaseId));
198+
return true;
199+
},
200+
);
201+
} finally {
202+
await runtime.shutdown();
203+
globalThis.fetch = previousFetch;
204+
}
205+
});
206+
207+
function makeDevice(lease: DeviceLease): DeviceInfo {
208+
return {
209+
...deviceFieldsFromPublicPlatform('android'),
210+
id: `webdriver-test:android:${lease.leaseId}`,
211+
name: 'Test device',
212+
kind: 'device',
213+
target: 'mobile',
214+
booted: true,
215+
};
216+
}
217+
50218
function makeLease(): DeviceLease {
51219
return {
52220
leaseId: 'lease-1',

‎packages/provider-webdriver/src/runtime-session.ts‎

Lines changed: 104 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,8 @@ import {
1010
createCloudWebDriverCapabilities,
1111
type CloudWebDriverProviderCapabilities,
1212
} from './capabilities.ts';
13-
import { WebDriverClient } from './webdriver-client.ts';
13+
import { WebDriverClient, type WebDriverSession } from './webdriver-client.ts';
14+
import { isWebDriverRequestTimeout } from './webdriver-transport.ts';
1415
import { createWebDriverInteractor } from './webdriver-interactor.ts';
1516
import { snapshotBackendForPlatform } from './runtime-helpers.ts';
1617
import type {
@@ -41,6 +42,8 @@ type CloudWebDriverCloseResult = Readonly<{
4142
warnings: CloudWebDriverReleaseWarning[];
4243
}>;
4344
type LeaseResult = Record<string, unknown> | undefined;
45+
/** The half of a provider session that exists once the WebDriver session does, registered or not. */
46+
type ProviderSessionHandle = Pick<WebDriverProviderSession, 'client' | 'prepared'>;
4447

4548
/**
4649
* Owns WebDriver session lifecycle and stale-owner retention. Deployment decisions stay in the
@@ -89,7 +92,7 @@ export class WebDriverSessionManager {
8992
headers: prepared.headers,
9093
requestPolicy: this.options.requestPolicy,
9194
});
92-
const session = await this.createSessionWithPreparedCleanup(client, prepared);
95+
const session = await this.createOwnedSession({ client, prepared }, lease, req);
9396
const device = this.deviceForLease(lease, prepared);
9497
const providerSessionId = prepared.providerSessionId ?? session.sessionId;
9598
const capabilities = createCloudWebDriverCapabilities({
@@ -164,18 +167,65 @@ export class WebDriverSessionManager {
164167
this.ownedDeviceIds.clear();
165168
}
166169

167-
private async createSessionWithPreparedCleanup(
168-
client: WebDriverClient,
169-
prepared: CloudWebDriverPreparedSession,
170-
): Promise<Awaited<ReturnType<WebDriverClient['createSession']>>> {
170+
/**
171+
* Creates the WebDriver session and settles who owns it. `POST /session` is
172+
* non-idempotent with an indeterminate outcome once abandoned (see
173+
* `WebDriverClient.createSession`), so the request's cancellation is treated
174+
* as ownership evidence rather than an interrupt: a requester that left
175+
* before creation started gets nothing created; one that left while the
176+
* provider was allocating gets the finished session released, because by
177+
* then its id is in hand and nothing else will ever release it (#1774).
178+
*/
179+
private async createOwnedSession(
180+
handle: ProviderSessionHandle,
181+
lease: DeviceLease,
182+
req: LeaseLifecycleContext | undefined,
183+
): Promise<WebDriverSession> {
184+
if (req?.signal?.aborted) {
185+
const canceled = requestCanceledError(this.options.provider, lease, {});
186+
await cleanupAfterCreateSessionFailure(handle.prepared, canceled);
187+
throw canceled;
188+
}
189+
const session = await this.createSessionOrCleanup(handle, lease, req);
190+
if (req?.signal?.aborted) {
191+
throw await this.releaseCanceledSession(handle, lease, session);
192+
}
193+
return session;
194+
}
195+
196+
private async createSessionOrCleanup(
197+
handle: ProviderSessionHandle,
198+
lease: DeviceLease,
199+
req: LeaseLifecycleContext | undefined,
200+
): Promise<WebDriverSession> {
171201
try {
172-
return await client.createSession(prepared.webdriverCapabilities);
202+
return await handle.client.createSession(handle.prepared.webdriverCapabilities, {
203+
deadline: req?.deadline,
204+
});
173205
} catch (error) {
174-
await cleanupAfterCreateSessionFailure(prepared, error);
175-
throw error;
206+
const failure = isWebDriverRequestTimeout(error)
207+
? sessionCreateTimeoutError(error, this.options.provider, lease, handle.prepared)
208+
: error;
209+
await cleanupAfterCreateSessionFailure(handle.prepared, failure);
210+
throw failure;
176211
}
177212
}
178213

214+
private async releaseCanceledSession(
215+
handle: ProviderSessionHandle,
216+
lease: DeviceLease,
217+
session: WebDriverSession,
218+
): Promise<AppError> {
219+
const close = await this.closeSession(handle);
220+
return requestCanceledError(this.options.provider, lease, {
221+
releasedWebDriverSessionId: session.sessionId,
222+
...(handle.prepared.providerSessionId
223+
? { releasedProviderSessionId: handle.prepared.providerSessionId }
224+
: {}),
225+
...(close.warnings.length > 0 ? { warnings: close.warnings } : {}),
226+
});
227+
}
228+
179229
private async prepareSession(
180230
lease: DeviceLease,
181231
req: LeaseLifecycleContext | undefined,
@@ -221,9 +271,7 @@ export class WebDriverSessionManager {
221271
};
222272
}
223273

224-
private async closeSession(
225-
session: WebDriverProviderSession,
226-
): Promise<CloudWebDriverCloseResult> {
274+
private async closeSession(session: ProviderSessionHandle): Promise<CloudWebDriverCloseResult> {
227275
const warnings: CloudWebDriverReleaseWarning[] = [];
228276
let cleanup: Record<string, unknown> | undefined;
229277
try {
@@ -296,3 +344,47 @@ async function cleanupAfterCreateSessionFailure(
296344
}
297345
}
298346
}
347+
348+
/**
349+
* The transport gave up on `POST /session`; the provider may still finish it.
350+
* Nothing here can learn that session's id, so the error carries what the
351+
* provider dashboard can be searched by instead — the lease the capabilities
352+
* were labelled with — rather than guessing at REST cleanup of a session this
353+
* process never owned.
354+
*/
355+
function sessionCreateTimeoutError(
356+
timeout: AppError,
357+
provider: string,
358+
lease: DeviceLease,
359+
prepared: CloudWebDriverPreparedSession,
360+
): AppError {
361+
const timeoutMs = timeout.details?.timeoutMs;
362+
return new AppError(
363+
'COMMAND_FAILED',
364+
`${provider} did not create the WebDriver session within ${String(timeoutMs)}ms.`,
365+
{
366+
reason: 'provider_session_create_timeout',
367+
provider,
368+
leaseId: lease.leaseId,
369+
runId: lease.runId,
370+
timeoutMs,
371+
...(prepared.providerSessionId ? { providerSessionId: prepared.providerSessionId } : {}),
372+
hint: `${provider} may still finish creating the session after agent-device stopped waiting, and that session would keep billing until the provider reaps it. Before retrying, check ${provider} for a running session created for lease ${lease.leaseId} (run ${lease.runId}) and stop it.`,
373+
},
374+
timeout,
375+
);
376+
}
377+
378+
function requestCanceledError(
379+
provider: string,
380+
lease: DeviceLease,
381+
released: Record<string, unknown>,
382+
): AppError {
383+
return new AppError('COMMAND_FAILED', 'request canceled', {
384+
reason: 'request_canceled',
385+
provider,
386+
leaseId: lease.leaseId,
387+
...released,
388+
hint: 'The lease request was canceled (explicit cancel or client disconnect) while the provider session was being created; the session it produced, if any, was released instead of registered.',
389+
});
390+
}

0 commit comments

Comments
 (0)