Repository navigation
Expand file tree
/
Copy pathrevyl-session.ts
More file actions
567 lines (537 loc) · 22.6 KB
/
Copy pathrevyl-session.ts
File metadata and controls
567 lines (537 loc) · 22.6 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
import { AppError, errorMessage } from '@agent-device/kernel/errors';
import { setTimeout as sleep } from 'node:timers/promises';
import type { DeviceLease } from '@agent-device/contracts/device';
import type { DeviceInfo } from '@agent-device/kernel/device';
import { buildRevylDevice, REVYL_SNAPSHOT_CHANNEL, type RevylPlatform } from './device.ts';
import {
isRevylSessionGone,
revylFailureRefusedTheRequest,
type RevylClient,
} from './revyl-client.ts';
import { revylSessionIdForLease } from './session-identity.ts';
import type {
RevylStartDeviceRequest,
RevylStartDeviceResponse,
RevylWorkerHealth,
} from './revyl-types.ts';
/** How long Revyl may take to assign a worker and connect the device before the start is given up on. */
const READY_TIMEOUT_MS = 240_000;
/** Revyl's worker answers `health` with 503 until the device is attached; the CLI polls on that beat. */
const READY_POLL_INTERVAL_MS = 2_000;
/** Revyl releases an idle session on its own; this is the lease the runtime asks for. */
const DEFAULT_IDLE_TIMEOUT_SECONDS = 30 * 60;
/** Revyl answers a device-proxy action the worker cannot serve yet with these. */
const NOT_READY_STATUSES = new Set([503, 409]);
export type RevylScreenSize = Readonly<{ width: number; height: number }>;
/** One live Revyl session: the run its actions route to, and the session that ends it. */
export type RevylSession = Readonly<{
platform: RevylPlatform;
/** The device-proxy routing key every action path is built from. */
runId: string;
/** What Revyl's `/stop` endpoint takes; the runtime issues it, so it is always known. */
sessionId: string;
/** Read from `health`; a snapshot without it cannot measure a keyboard band. */
screen: RevylScreenSize | undefined;
device: DeviceInfo;
/** The channel a `hierarchy` answer is parsed and presented on for this platform. */
snapshotChannel: (typeof REVYL_SNAPSHOT_CHANNEL)[RevylPlatform];
client: RevylClient;
lease: DeviceLease;
/**
* The app this session last launched, which is the only one Revyl's `/kill_app` can end: that action
* carries no bundle id. Held in a box so the session itself stays a frozen value.
*/
launchedApp: { bundleId?: string };
}>;
export type RevylStartRequestInput = Readonly<{
platform: RevylPlatform;
appUrl: string | undefined;
deviceModel: string | undefined;
osVersion: string | undefined;
signal?: AbortSignal;
/** Epoch-ms deadline the caller has for the whole allocation; it outranks the runtime's own. */
deadline?: number;
/**
* Receives the work that ends a start the caller stopped waiting for, which runs on after this call has
* already failed. A runtime that can still release the lease takes it, so its release or shutdown waits
* for that device instead of reporting a release it never made.
*/
onLateStart?: (end: Promise<RevylLateStartEnd | undefined>) => void;
}>;
/**
* What ending a start that landed late reports: the session it addressed, and whether Revyl confirmed it
* is gone. Absent when the answer showed nothing was rented.
*/
export type RevylLateStartEnd = Readonly<{ sessionId: string; confirmed: boolean }>;
/**
* One caller's own patience: the signal it cancels with and the deadline it stops at. Both are optional,
* and a caller with neither waits for whatever its work takes.
*/
export type RevylCallerBudget = Readonly<{ signal?: AbortSignal; deadline?: number }>;
/**
* One caller's wait for work this process is already doing, bounded by that caller's own deadline and
* canceled with its own signal. `settled` says whether this wait is what ended the race, which is what
* lets a caller report its own cancellation rather than the outcome of a shared request it stopped
* watching. `dispose` retires the listener once the work itself is done.
*/
export type RevylCallerWait<T> = Readonly<{
promise: Promise<T>;
settled: () => boolean;
dispose: () => void;
}>;
/**
* What one start has to undo: the session that holds the rented device, and the run that routes to
* it. `sessionId` is always the id the backend was told, or the one it answered with, so the end
* always addresses what Revyl actually holds.
*/
type RevylStartCleanup = Readonly<{ runId?: string; sessionId: string }>;
/**
* Starts a Revyl device and waits until it answers: `start_device`, then the worker-connection poll,
* then `health` until the device reports connected. A start that never became ready is stopped before
* the failure reaches the caller, so a failed allocate never leaves a rented device running — and a
* stop that failed is named in the failure, because the device is still rented.
*/
export async function startRevylSession(
client: RevylClient,
lease: DeviceLease,
input: RevylStartRequestInput,
): Promise<RevylSession> {
// The runtime issues the session id rather than reading it back, so a session this lease started
// can be ended from the lease alone — including from a process that never started it.
const sessionId = revylSessionIdForLease(lease.leaseId);
const request: RevylStartDeviceRequest = {
platform: input.platform,
is_simulation: true,
session_id: sessionId,
idle_timeout_seconds: DEFAULT_IDLE_TIMEOUT_SECONDS,
...(input.appUrl === undefined ? {} : { app_url: input.appUrl }),
...(input.deviceModel === undefined ? {} : { device_model: input.deviceModel }),
...(input.osVersion === undefined ? {} : { os_version: input.osVersion }),
};
const started = await submitRevylStart(client, request, sessionId, input);
const runId = typeof started.workflow_run_id === 'string' ? started.workflow_run_id : undefined;
const echoed = typeof started.session_id === 'string' ? started.session_id : undefined;
if (runId && echoed && echoed !== sessionId) {
// Every later stop addresses the session by the id the runtime issued, and an expired-lease
// recovery in another process derives that same id from the lease. A backend that kept its own
// identity therefore has to be refused, and the id it named is the only handle that still names
// the device: that is what gets stopped and what the failure reports.
const cleanup: RevylStartCleanup = { runId, sessionId: echoed };
throwIfStartStopFailed(
cleanup,
await stopStartedSession(client, cleanup),
new AppError(
'COMMAND_FAILED',
`Revyl started the device as session ${echoed} instead of ${sessionId}.`,
{
reason: 'provider-session-identity-mismatch',
runId,
providerSessionId: echoed,
hint: 'Retry the command; agent-device ends a Revyl session by the identity it issued.',
},
),
);
}
if (!runId) {
// Revyl opened a session and only failed to hand back a run. The id it echoes names that session
// when it kept its own, so cleanup addresses the session the backend actually holds.
const cleanup: RevylStartCleanup = { sessionId: echoed ?? sessionId };
throwIfStartStopFailed(
cleanup,
await stopStartedSession(client, cleanup),
new AppError(
'COMMAND_FAILED',
`Revyl did not start a ${input.platform} device: ${started.error ?? started.message ?? 'no workflow run was returned'}`,
{
reason: 'provider-start-refused',
platform: input.platform,
...(started.trace_id === undefined ? {} : { traceId: started.trace_id }),
},
),
);
}
// Revyl may have answered a run for a start the caller already gave up on. That device is rented
// either way, so it is stopped here before the cancellation reaches the caller.
if (input.signal?.aborted) {
const cleanup: RevylStartCleanup = { runId, sessionId };
throwIfStartStopFailed(
cleanup,
await stopStartedSession(client, cleanup),
startCanceled(input),
);
}
const deadline = readinessDeadline(input);
try {
const health = await awaitRevylDeviceReady(client, runId, deadline, input);
return {
platform: input.platform,
runId,
sessionId,
screen: readRevylScreenSize(health),
device: buildRevylDevice(input.platform, lease, runId),
snapshotChannel: REVYL_SNAPSHOT_CHANNEL[input.platform],
client,
lease,
launchedApp: {},
};
} catch (error) {
const cleanup: RevylStartCleanup = { runId, sessionId };
throwIfStartStopFailed(cleanup, await stopStartedSession(client, cleanup), error);
}
}
/**
* Sends `start_device` within the caller's budget. The request rents a device even when the caller
* hangs up, so two things keep that from leaking: the request keeps the client's own start budget rather
* than the caller's shorter one, so an answer still comes back, and the session id is issued before the
* request goes out, so an answer nobody read still leaves a session that can be stopped by that id. The
* caller's cancellation never travels on the request itself: aborting the fetch discards the answer, and
* an answer nobody read is a rented device with no run id.
*/
async function submitRevylStart(
client: RevylClient,
request: RevylStartDeviceRequest,
sessionId: string,
input: RevylStartRequestInput,
): Promise<RevylStartDeviceResponse> {
if (input.signal?.aborted) throw startCanceled(input);
const callerDeadline = input.deadline ?? Number.POSITIVE_INFINITY;
// No signal of the caller's travels on the request, and the caller's shorter deadline is not put on it
// either: the request keeps the client's own start budget, so a caller that stops waiting still leaves
// an answer to come back, and that answer is the only thing that names the rented device.
const start = client.startDevice(request);
const gaveUp = callerWait(callerDeadline, input);
try {
return await (gaveUp === undefined ? start : Promise.race([start, gaveUp.promise]));
} catch (error) {
if (gaveUp?.settled() === true) {
// The caller stopped waiting, and the request is still owed an answer that names the device. A stop
// sent now could only address the id this side issued, which a backend that has not created the
// session yet answers as gone, so the request is left running and what it returns is what gets ended.
input.onLateStart?.(endStartTheCallerGaveUpOn(client, sessionId, start));
throw startCanceled(input);
}
// The request went out and never reached the caller, so something may be rented under the id it
// carried; that id is the handle, and the caller's failure stays the reported one.
if (startMayHaveRented(error)) {
const cleanup: RevylStartCleanup = { sessionId };
throwIfStartStopFailed(cleanup, await stopStartedSession(client, cleanup), error);
}
throw error;
} finally {
gaveUp?.dispose();
}
}
/**
* Ends a start the caller stopped waiting for, once Revyl has said what it made. The answer names the
* device, so the request is not discarded; a start Revyl refused with a settled verdict rented nothing to
* undo, and one that never answered at all is ended by the id the request carried. The caller has already
* been answered with its own cancellation, so the work is handed to `onLateStart` rather than reported.
*/
async function endStartTheCallerGaveUpOn(
client: RevylClient,
sessionId: string,
start: Promise<RevylStartDeviceResponse>,
): Promise<RevylLateStartEnd | undefined> {
const cleanup = await start.then(
(started) => {
const runId =
typeof started.workflow_run_id === 'string' ? started.workflow_run_id : undefined;
const echoed = typeof started.session_id === 'string' ? started.session_id : undefined;
return { ...(runId === undefined ? {} : { runId }), sessionId: echoed ?? sessionId };
},
(error: unknown) =>
// Revyl settled the request with a refusal, so nothing was rented.
startMayHaveRented(error) ? { sessionId } : undefined,
);
if (cleanup === undefined) return undefined;
const failure = await stopStartedSession(client, cleanup);
return { sessionId: cleanup.sessionId, confirmed: failure === undefined };
}
/**
* One caller's wait for work this process is already doing, bounded by that caller's own deadline and
* canceled with its own signal. `settled` says whether this wait is what ended the race, which is what
* lets a caller report its own cancellation rather than the outcome of a shared request it stopped
* watching. `dispose` retires the listener once the work itself is done.
*/
export function revylCallerWait<T>(
deadline: number,
budget: RevylCallerBudget,
onGiveUp: () => unknown,
): RevylCallerWait<T> | undefined {
if (budget.deadline === undefined && budget.signal === undefined) return undefined;
const parts: AbortSignal[] = [];
if (Number.isFinite(deadline))
parts.push(AbortSignal.timeout(Math.max(0, deadline - Date.now())));
if (budget.signal !== undefined) parts.push(budget.signal);
const signal = parts.length === 1 ? parts[0]! : AbortSignal.any(parts);
let rejected = false;
let onAbort: () => void = () => {};
const reject = (reason: unknown) => {
rejected = true;
return reason;
};
// A signal that is already aborted never fires its listener, so that wait has to start over the
// edge rather than sit on it.
const promise = signal.aborted
? Promise.reject(reject(onGiveUp()))
: new Promise<T>((_, rejectPromise) => {
onAbort = () => rejectPromise(reject(onGiveUp()));
signal.addEventListener('abort', onAbort, { once: true });
});
return Object.freeze({
promise,
settled: () => rejected,
dispose: () => signal.removeEventListener('abort', onAbort),
});
}
/**
* The caller's own wait for the start, or absent when the caller gave it neither a deadline nor a
* signal and the request's timeout is all there is to wait for.
*/
function callerWait(
deadline: number,
input: RevylStartRequestInput,
): RevylCallerWait<RevylStartDeviceResponse> | undefined {
return revylCallerWait<RevylStartDeviceResponse>(deadline, input, () => startCanceled(input));
}
function startCanceled(input: RevylStartRequestInput): AppError {
return new AppError(
'COMMAND_FAILED',
'The Revyl device start was canceled before it was ready.',
{ reason: 'request_canceled', platform: input.platform },
input.signal?.reason,
);
}
/**
* The failure for a caller that stopped waiting for a device: its own cancellation when it hung up, and
* the budget that ran out when it did not. `budgetMs` is the wait that ended, which is the number the
* report has to name — the readiness phase's own budget there, the caller's whole deadline for a join
* that never had a phase of its own.
*/
export function revylCallerGaveUp(
platform: RevylPlatform,
budget: RevylCallerBudget,
budgetMs: number,
): AppError {
if (budget.signal?.aborted) {
return new AppError(
'COMMAND_FAILED',
'The Revyl device start was canceled before it was ready.',
{ reason: 'request_canceled', platform },
budget.signal.reason,
);
}
return new AppError(
'COMMAND_FAILED',
`Revyl did not report a ready device within ${budgetMs} ms.`,
{
reason: 'provider-not-ready',
timeoutMs: budgetMs,
...(budget.deadline === undefined ? {} : { deadline: budget.deadline }),
hint: 'Retry the command; Revyl may have had no device of that model available.',
},
);
}
/**
* Whether a `start_device` that never reached the caller may still have rented a device. Only a status
* the backend settled with a refusal rules that out, and that is one read of the client's status table.
* A lost answer, an unreachable backend, an unconfirmed status, or the caller hanging up all leave the
* session possibly open under the id the request carried, which is the id this module issued.
*/
function startMayHaveRented(error: unknown): boolean {
return !revylFailureRefusedTheRequest(error);
}
/**
* Ends what one start rented, and reports the failure instead of swallowing it: a caller that cannot
* reach Revyl has to hear that the device it asked for is still rented. A session Revyl no longer has
* is the success this path wants, so it is not a failure.
*/
async function stopStartedSession(
client: RevylClient,
cleanup: RevylStartCleanup,
): Promise<unknown | undefined> {
try {
await stopRevylSession(client, cleanup.runId, cleanup.sessionId);
return undefined;
} catch (error) {
return isRevylSessionGone(error) ? undefined : error;
}
}
/**
* Reports a rented device the start could not hand back. The caller's own failure becomes the cause:
* what is still billed outranks why the start did not finish, and the session id in the details is
* what has to be ended by hand.
*/
function throwIfStartStopFailed(
cleanup: RevylStartCleanup,
stopFailure: unknown | undefined,
startFailure: unknown,
): never {
if (stopFailure === undefined) throw startFailure;
throw new AppError(
'COMMAND_FAILED',
`Revyl did not release the session it started (${cleanup.sessionId}).`,
{
reason: 'provider-cleanup-pending',
providerSessionId: cleanup.sessionId,
...(cleanup.runId === undefined ? {} : { runId: cleanup.runId }),
cause: errorMessage(stopFailure),
hint: `End the Revyl session ${cleanup.sessionId}; the device stays rented and billed until it is.`,
},
startFailure,
);
}
/** The readiness budget: the instant it runs out, and how much of it the caller was given. */
type RevylDeadline = Readonly<{ at: number; budgetMs: number }>;
/** The caller's own budget wins when it is shorter, so a start never outlives the lease request. */
function readinessDeadline(input: RevylStartRequestInput): RevylDeadline {
const remaining =
input.deadline === undefined
? READY_TIMEOUT_MS
: Math.min(READY_TIMEOUT_MS, input.deadline - Date.now());
return { at: Date.now() + remaining, budgetMs: Math.max(0, remaining) };
}
async function awaitRevylDeviceReady(
client: RevylClient,
runId: string,
deadline: RevylDeadline,
input: RevylStartRequestInput,
): Promise<RevylWorkerHealth> {
await untilRevylWorkerReady(client, runId, deadline, input);
for (;;) {
requireTime(deadline, input);
const response = await readWorkerHealth(client, runId, deadline, input);
// The deadline is checked again on the answer's heels: a late success still missed it.
requireTime(deadline, input);
if (response.device_connected === true) return response;
await sleepBefore(deadline, input.signal);
}
}
/**
* One `health` read. Revyl answers it with 503, and the worker with 409, while the device is not
* attached yet, which is the same not-ready a `device_connected: false` answer carries.
*/
async function readWorkerHealth(
client: RevylClient,
runId: string,
deadline: RevylDeadline,
input: RevylStartRequestInput,
): Promise<RevylWorkerHealth> {
try {
return await runWithinDeadline(
() => client.getWorkerHealth(runId, pollSignal(deadline, input)),
deadline,
input,
);
} catch (error) {
if (!isProviderNotReady(error)) throw error;
return {};
}
}
/**
* Runs one readiness poll and reports a failure the deadline explains as the deadline's own answer.
* A poll the budget cut short reaches the client as a cancellation, and a caller that never canceled
* has to hear that Revyl was too slow rather than that the request was canceled.
*/
async function runWithinDeadline<T>(
poll: () => Promise<T>,
deadline: RevylDeadline,
input: RevylStartRequestInput,
): Promise<T> {
try {
return await poll();
} catch (error) {
requireTime(deadline, input);
throw error;
}
}
/** Revyl has not served this action yet, so the reason is transient and the poll continues. */
function isProviderNotReady(error: unknown): boolean {
return (
error instanceof AppError &&
typeof error.details?.providerStatus === 'number' &&
NOT_READY_STATUSES.has(error.details.providerStatus)
);
}
/** The worker-connection poll. `cancelled`, `stopped` and `failed` are terminal: no later poll is ready. */
async function untilRevylWorkerReady(
client: RevylClient,
runId: string,
deadline: RevylDeadline,
input: RevylStartRequestInput,
): Promise<void> {
for (;;) {
requireTime(deadline, input);
const connection = await runWithinDeadline(
() => client.getWorkerConnection(runId, pollSignal(deadline, input)),
deadline,
input,
);
requireTime(deadline, input);
if (connection.status === 'ready' && connection.worker_ws_url) return;
if (
connection.status === 'cancelled' ||
connection.status === 'stopped' ||
connection.status === 'failed'
) {
throw new AppError(
'COMMAND_FAILED',
`Revyl did not start the ${input.platform} device: ${connection.status}${
connection.message === null || connection.message === undefined
? ''
: ` (${connection.message})`
}`,
{ reason: 'provider-start-failed', runId, status: connection.status },
);
}
await sleepBefore(deadline, input.signal);
}
}
/** Fails the moment the readiness budget is spent, whatever a poll that was already sent answers. */
function requireTime(deadline: RevylDeadline, input: RevylStartRequestInput): void {
if (Date.now() < deadline.at) return;
// The budget is the caller's deadline when it is shorter, so the number named has to be the one that
// actually ran out.
throw revylCallerGaveUp(input.platform, input, deadline.budgetMs);
}
/** The remaining readiness budget, so a poll that outlives it is canceled rather than awaited. */
function pollSignal(deadline: RevylDeadline, input: RevylStartRequestInput): AbortSignal {
const remaining = AbortSignal.timeout(Math.max(0, deadline.at - Date.now()));
return input.signal === undefined ? remaining : AbortSignal.any([input.signal, remaining]);
}
async function sleepBefore(deadline: RevylDeadline, signal?: AbortSignal): Promise<void> {
await sleep(
Math.min(READY_POLL_INTERVAL_MS, Math.max(0, deadline.at - Date.now())),
undefined,
signal === undefined ? {} : { signal },
).catch(() => undefined);
}
/**
* Ends one Revyl session. `/stop` is idempotent, so a repeat release is safe, and a session with no
* run to route to is ended by its session id — the same pair of ends Revyl's own clients take.
*/
export async function stopRevylSession(
client: RevylClient,
runId: string | undefined,
sessionId: string | undefined,
): Promise<void> {
if (sessionId) {
await client.stopDeviceSession(sessionId);
return;
}
if (runId) await client.cancelDeviceRun(runId);
}
function readRevylScreenSize(health: RevylWorkerHealth): RevylScreenSize | undefined {
const { screen_width: width, screen_height: height } = health;
if (
typeof width !== 'number' ||
typeof height !== 'number' ||
!Number.isFinite(width) ||
!Number.isFinite(height) ||
width <= 0 ||
height <= 0
) {
return undefined;
}
return { width, height };
}