Skip to content

Commit b942ccd

Browse files
committed
feat(provider-webos): add LG webOS TV provider plugin
Add @agent-device/webos, an optional provider plugin that drives web apps on LG webOS TVs, and the private @agent-device/tv-cdp package it bundles, which holds the CDP page layer a later Tizen plugin will share. The TV enters as an Android TV device owned by the plugin (id webos:<ares device>). Lifecycle runs through ares-launch, and the page is driven over the Chrome DevTools Protocol through the ares-inspect tunnel. v1 covers open, close, apps, snapshot, remote keys (directions, select, back), pointer, text, scroll, screenshot, logs and appstate. Releasing a lease closes the page and the tunnel, and a page whose socket, tunnel or app goes away is detached, so later commands ask for open again. - Kernel: add the cdp-dom snapshot producer on the android channel so DOM snapshots do not pose as UIAutomator output. - Capture kit: let shapeDesktopSurfaceSnapshot take an interactive test, since every DOM node carries a rect. - Packed-install smoke: let scripts/check-provider-plugin.mjs run fixtures for providers without credentials, and add the webOS fixture. - Docs: add the LG webOS page, mention it in plugins, commands and the sidebar, and add a webOS flow to help remote.
1 parent a0e7c7d commit b942ccd

45 files changed

Lines changed: 2427 additions & 37 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.fallowrc.json‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@
1111
"packages/provider-revyl/test/package-smoke.mjs",
1212
"packages/provider-revyl/tsdown.config.ts",
1313
"packages/provider-revyl/src/plugin.ts",
14+
"packages/provider-webos/test/package-smoke.mjs",
15+
"packages/provider-webos/tsdown.config.ts",
1416
"vitest.mutation.config.ts",
1517
"src/sdk/index.ts",
1618
"src/sdk/io.ts",
@@ -98,6 +100,10 @@
98100
// through this workspace link to core source so typecheck needs no build; Fallow then credits
99101
// the source files rather than the package.
100102
"agent-device",
103+
// Nothing in the root imports the webOS plugin or the tv-cdp package it bundles, but layering
104+
// requires the root manifest to declare every workspace package.
105+
"@agent-device/tv-cdp",
106+
"@agent-device/webos",
101107
// Oxlint resolves the shared config's JS plugins dynamically from their package names.
102108
"@nkzw/eslint-plugin",
103109
"eslint-plugin-no-only-tests",

‎examples/sdk/tsconfig.json‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@
6161
{ "path": "../../packages/provider-limrun" },
6262
{ "path": "../../packages/provider-revyl" },
6363
{ "path": "../../packages/provider-testmu" },
64+
{ "path": "../../packages/provider-webos" },
6465
{ "path": "../../packages/provider-webdriver" },
6566
{ "path": "../../packages/provision-kit" },
6667
{ "path": "../../packages/proxy" },
@@ -71,6 +72,7 @@
7172
{ "path": "../../packages/selectors" },
7273
{ "path": "../../packages/session-journal" },
7374
{ "path": "../../packages/test-kit" },
75+
{ "path": "../../packages/tv-cdp" },
7476
{ "path": "../../packages/xml" }
7577
]
7678
}

‎package.json‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -336,6 +336,7 @@
336336
"@agent-device/selectors": "workspace:*",
337337
"@agent-device/session-journal": "workspace:*",
338338
"@agent-device/test-kit": "workspace:*",
339+
"@agent-device/tv-cdp": "workspace:*",
339340
"@agent-device/xml": "workspace:*",
340341
"@arethetypeswrong/cli": "^0.18.5",
341342
"@chenglou/freerange": "^0.0.4",
@@ -375,6 +376,7 @@
375376
"yaml": "^2.9.0",
376377
"yauzl": "^3.4.0",
377378
"@agent-device/testmu": "workspace:*",
378-
"@agent-device/revyl": "workspace:*"
379+
"@agent-device/revyl": "workspace:*",
380+
"@agent-device/webos": "workspace:*"
379381
}
380382
}

‎packages/capture-kit/src/snapshot-desktop-projection.ts‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,19 +16,19 @@ const INTERACTIVE_ROLE_TOKENS = [
1616
'switch',
1717
] as const;
1818

19+
/** `isInteractive` overrides the default test for producers whose every node carries a rect. */
1920
export function shapeDesktopSurfaceSnapshot(
2021
data: SnapshotResult,
2122
options: Pick<SnapshotOptions, 'depth' | 'interactiveOnly' | 'scope'>,
23+
isInteractive: (node: RawSnapshotNode) => boolean = isInteractiveSnapshotNode,
2224
): SnapshotResult {
2325
let nodes = data.nodes ?? [];
2426
if (options.scope) {
2527
nodes = scopeSnapshotNodes(nodes, options.scope, (range) =>
26-
options.interactiveOnly
27-
? nodes.slice(range.start, range.end).some(isInteractiveSnapshotNode)
28-
: true,
28+
options.interactiveOnly ? nodes.slice(range.start, range.end).some(isInteractive) : true,
2929
);
3030
}
31-
if (options.interactiveOnly) nodes = filterInteractiveSnapshotNodes(nodes);
31+
if (options.interactiveOnly) nodes = filterInteractiveSnapshotNodes(nodes, isInteractive);
3232
if (typeof options.depth === 'number') nodes = filterSnapshotNodesByDepth(nodes, options.depth);
3333
return { ...data, nodes };
3434
}
@@ -46,12 +46,15 @@ export function scopeSnapshotNodes(
4646
return reindexSnapshotNodes(slice, slice[0]?.depth ?? 0);
4747
}
4848

49-
function filterInteractiveSnapshotNodes(nodes: RawSnapshotNode[]): RawSnapshotNode[] {
49+
function filterInteractiveSnapshotNodes(
50+
nodes: RawSnapshotNode[],
51+
isInteractive: (node: RawSnapshotNode) => boolean,
52+
): RawSnapshotNode[] {
5053
if (nodes.length === 0) return nodes;
5154
const byIndex = new Map(nodes.map((node) => [node.index, node]));
5255
const keepIndexes = new Set<number>();
5356
for (const node of nodes) {
54-
if (!isInteractiveSnapshotNode(node)) continue;
57+
if (!isInteractive(node)) continue;
5558
let current: RawSnapshotNode | undefined = node;
5659
while (current) {
5760
if (keepIndexes.has(current.index)) break;

‎packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@ const DEFAULT_QUIET_MS_BY_PRODUCER = {
6767
'limrun-ios-tree': DEFAULT_STABLE_QUIET_MS,
6868
'revyl-ios-tree': DEFAULT_STABLE_QUIET_MS,
6969
'android-uiautomator': DEFAULT_STABLE_QUIET_MS,
70+
'cdp-dom': DEFAULT_STABLE_QUIET_MS,
7071
'harmonyos-uitest': DEFAULT_STABLE_QUIET_MS,
7172
'macos-helper': DEFAULT_STABLE_QUIET_MS,
7273
'linux-atspi': DEFAULT_STABLE_QUIET_MS,

‎packages/kernel/src/snapshot.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -587,7 +587,7 @@ export type SnapshotProvenance =
587587
| 'limrun-ios-tree'
588588
| 'revyl-ios-tree';
589589
}
590-
| { backend: 'android'; producer: 'android-uiautomator' | 'appium-source' }
590+
| { backend: 'android'; producer: 'android-uiautomator' | 'appium-source' | 'cdp-dom' }
591591
| { backend: 'harmonyos-arkui'; producer: 'harmonyos-uitest' }
592592
| { backend: 'macos-helper'; producer: 'macos-helper' }
593593
| { backend: 'linux-atspi'; producer: 'linux-atspi' }

‎packages/provider-testmu/test/package-smoke.mjs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
export const provider = 'testmu';
12
export const args = [
23
'--platform',
34
'android',

‎packages/provider-webos/README.md‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
# @agent-device/webos
2+
3+
Drive web apps on LG webOS TVs with [agent-device](https://agent-device.dev). The plugin launches
4+
the app with the webOS CLI, opens the `ares-inspect` tunnel, and drives the page over the Chrome
5+
DevTools Protocol: remote keys, pointer, text, snapshots, screenshots and console logs.
6+
7+
Status: experimental. Verified on an LG 32LQ63006LA (webOS TV 22, dev build of a React Native Web
8+
app): `connect`, `open`, `snapshot -i`, `snapshot -s`, `tv-remote` (directions, select, back),
9+
`is focused`, `press @ref --settle`, `wait text`, `find`, `screenshot`, `logs`, `apps`,
10+
`appstate`, `close`, `disconnect`.
11+
12+
## How it plugs in
13+
14+
agent-device leases only iOS, Android and HarmonyOS devices, so a webOS TV enters as an
15+
**Android TV device that this plugin owns** (`platform: android`, `target: tv`, id
16+
`webos:<ares device>`). Every command on that device goes to this plugin's runtime, never to adb.
17+
18+
## Requirements
19+
20+
- webOS CLI (`ares-*`) on `PATH`, or the directory that holds it passed as `aresBin`; see LG's
21+
[webOS CLI installation](https://webostv.developer.lge.com/develop/tools/cli-installation)
22+
- The TV in Developer Mode and registered with `ares-setup-device`; see LG's
23+
[App Testing with Developer Mode App](https://webostv.developer.lge.com/develop/getting-started/developer-mode-app)
24+
- The app installed on the TV
25+
- The TV and this host on the same network
26+
27+
## Install
28+
29+
```sh
30+
agent-device plugins add @agent-device/webos
31+
agent-device daemon stop # the next command starts a daemon with the plugin loaded
32+
```
33+
34+
Plugin options: `aresBin` (directory with the `ares-*` CLIs) and `device` (default ares device
35+
name).
36+
37+
To try a local build, pack the package and install the tarball into a plugin store installation:
38+
39+
```sh
40+
pnpm --filter @agent-device/webos pack --out /tmp/agent-device-webos.tgz
41+
installation=$(uuidgen)
42+
npm install --prefix ~/.agent-device/plugins/$installation /tmp/agent-device-webos.tgz
43+
```
44+
45+
Then register `{"plugins": {"@agent-device/webos": {"installation": "<installation>"}}}` in
46+
`~/.agent-device/config.json` and run `agent-device daemon stop`.
47+
48+
## Use
49+
50+
```sh
51+
agent-device daemon stop # the lease TTL is read when the daemon starts
52+
export AGENT_DEVICE_LEASE_TTL_MS=1800000 AGENT_DEVICE_LEASE_MAX_TTL_MS=1800000 # see Known limitations
53+
agent-device connect webos [--device <ares device>] # defaults to the ares default device
54+
agent-device open <appId> # ares-launch, then attach over CDP
55+
agent-device tv-remote press down
56+
agent-device snapshot -i
57+
agent-device screenshot ./tv.png
58+
agent-device logs start # then logs path
59+
agent-device close <appId>
60+
agent-device disconnect
61+
```
62+
63+
Later commands use the session `connect` created; `connect` prints its name for `--session`.
64+
65+
## Supported
66+
67+
| Command | How |
68+
| --- | --- |
69+
| `open` / `close` | `ares-launch` / `ares-launch --close`, then the `ares-inspect` tunnel |
70+
| `tv-remote press up/down/left/right/select/back`, `back` | `Input.dispatchKeyEvent` with the LG remote codes (Back = 461) |
71+
| `appstate` | the attached app while its page is visible |
72+
| `apps` | `ares-install --listfull` |
73+
| `logs` | console calls, exceptions and browser log entries of the attached page, one line each |
74+
| `snapshot`, `find`, `is`, `get`, `wait` | DOM walk over `Runtime.evaluate`: visible interactive, labelled or text nodes, focus from `document.activeElement`; `-s`, `-d` and `-i` apply to the walk |
75+
| `screenshot` | `Page.captureScreenshot` |
76+
| `click`, `press <x> <y>`, `longpress`, `hover` | `Input.dispatchMouseEvent` (Magic Remote pointer), ending each move with a small wiggle that wakes the cursor |
77+
| `fill` | pointer press on the field, select its contents, then `Input.insertText`, or Backspace for an empty value |
78+
| `type` | `Input.insertText` into the focused element |
79+
| `scroll` | `mouseWheel` at the centre of the scroll container or viewport, by the requested distance |
80+
81+
Not yet: system buttons (`home`, menu, volume, channel and media keys), `network dump`, `perf`,
82+
`install` and `uninstall`, recording, URLs and deep links in `open`, `power`.
83+
84+
## Known limitations
85+
86+
- **Idle leases expire after 60 s by default, and expiry ends the session.** agent-device gives a
87+
direct-device provider lease its default TTL and beats it only while a command runs. When the
88+
lease ends, by `disconnect` or by expiry, the plugin closes the page connection and the
89+
`ares-inspect` tunnel. After a longer pause, the next command fails with `Lease is not active`
90+
or `No active session` and needs a new `connect` and `open`. To avoid it, run
91+
`agent-device daemon stop`, export `AGENT_DEVICE_LEASE_TTL_MS` and
92+
`AGENT_DEVICE_LEASE_MAX_TTL_MS` (for example `1800000`, 30 min), and run the next command in
93+
that shell. That command starts the daemon with both values.
94+
- **`screenshot` captures the app page, not the TV screen.** When the app is in the background,
95+
for example after Home on the remote, the page is hidden and renders no frames, so `screenshot`
96+
fails at once; `open <appId>` brings the app back. A whole-screen capture is not reachable on a retail TV: SSAP
97+
`tv/executeOneShot` answers `401 insufficient permissions` to unsigned clients, LG rejects the
98+
public signed manifest (`blacklisted certificate`), and the dev-mode shell (`prisoner`) gets no
99+
answer from `luna-send-pub`.
100+
- **Console messages from before the attach are replayed without object previews**, so objects
101+
in them show as `Object`. Messages logged after the attach show their properties.
102+
- **`open` on a running app can reload it** (webOS relaunch), so the first read may hit a document
103+
without a body. `open` waits for a loaded document, and a snapshot of an unloaded page is empty
104+
rather than an error. `open --relaunch` closes the app first, for a cold start.
105+
- **A second DevTools client on the same page hung in testing.** Close Chrome DevTools and other
106+
inspector clients attached to the app before a session.
107+
- **Snapshots are DOM-derived.** Labels come from `aria-label`, `alt`, `title` or own text;
108+
rects are CSS px, which match screenshot pixels only at `devicePixelRatio` 1 (1080p apps).
109+
The walk does not enter iframes or shadow roots, so overlays rendered there (for example a
110+
dev-server error overlay) appear in screenshots but not in snapshots.
111+
- **System overlays are outside the page.** The LG menu, on-screen keyboard and other system
112+
layers never appear in `screenshot` or `snapshot`.
Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
{
2+
"name": "@agent-device/webos",
3+
"version": "0.21.36-dev",
4+
"type": "module",
5+
"description": "LG webOS TV web app plugin for agent-device.",
6+
"files": [
7+
"dist"
8+
],
9+
"scripts": {
10+
"build": "tsdown --config tsdown.config.ts",
11+
"prepack": "pnpm build"
12+
},
13+
"exports": {
14+
".": {
15+
"types": "./src/index.ts",
16+
"default": "./src/index.ts"
17+
},
18+
"./agent-device": {
19+
"types": "./src/index.ts",
20+
"default": "./src/index.ts"
21+
},
22+
"./package.json": "./package.json"
23+
},
24+
"devDependencies": {
25+
"@agent-device/host-kit": "workspace:*",
26+
"@agent-device/kernel": "workspace:*",
27+
"@agent-device/test-kit": "workspace:*",
28+
"@agent-device/tv-cdp": "workspace:*",
29+
"agent-device": "workspace:*"
30+
},
31+
"publishConfig": {
32+
"exports": {
33+
".": {
34+
"import": "./dist/agent-device.mjs"
35+
},
36+
"./agent-device": {
37+
"import": "./dist/agent-device.mjs"
38+
}
39+
},
40+
"access": "public"
41+
},
42+
"license": "MIT",
43+
"repository": {
44+
"type": "git",
45+
"url": "git+https://github.com/callstack/agent-device.git",
46+
"directory": "packages/provider-webos"
47+
},
48+
"homepage": "https://agent-device.dev/docs/webos"
49+
}
Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
import assert from 'node:assert/strict';
2+
import { chmodSync, writeFileSync } from 'node:fs';
3+
import path from 'node:path';
4+
import { isRequestCanceledError } from '@agent-device/kernel/errors';
5+
import { mkdtempForTestSync } from '@agent-device/test-kit';
6+
import { test } from 'vitest';
7+
8+
import { createAres } from './ares.ts';
9+
10+
function fakeAresBin(outputs: Record<string, string>) {
11+
const bin = mkdtempForTestSync('ares-');
12+
for (const [name, output] of Object.entries(outputs)) {
13+
const script = path.join(bin, name);
14+
writeFileSync(script, `#!/bin/sh\ncat <<'OUTPUT'\n${output}\nOUTPUT\n`);
15+
chmodSync(script, 0o755);
16+
}
17+
return bin;
18+
}
19+
20+
test('listDevices returns only TV profiles with their IP and default flag', async () => {
21+
const bin = fakeAresBin({
22+
'ares-setup-device': JSON.stringify([
23+
{ profile: 'tv', name: 'living-room', default: true, deviceinfo: { ip: '192.0.2.10' } },
24+
{ profile: 'ose', name: 'signage', default: false, deviceinfo: { ip: '192.0.2.20' } },
25+
]),
26+
});
27+
28+
const devices = await createAres(bin, process.env).listDevices();
29+
30+
assert.deepEqual(devices, [{ name: 'living-room', isDefault: true }]);
31+
});
32+
33+
test('listApps reads each app id with its title from ares-install --listfull', async () => {
34+
const bin = fakeAresBin({
35+
'ares-install': [
36+
'[Info] Set target device : living-room',
37+
'id : com.sdk.ares.hostedapp',
38+
'title : Ares Hosted App',
39+
'version : 1.1.0',
40+
'',
41+
'id : com.example.tvapp',
42+
'title : Example TV',
43+
'requiredACG',
44+
'',
45+
'id : com.example.untitled',
46+
].join('\n'),
47+
});
48+
49+
const apps = await createAres(bin, process.env).forDevice('living-room').listApps();
50+
51+
assert.deepEqual(apps, [
52+
{ id: 'com.sdk.ares.hostedapp', name: 'Ares Hosted App' },
53+
{ id: 'com.example.tvapp', name: 'Example TV' },
54+
{ id: 'com.example.untitled', name: 'com.example.untitled' },
55+
]);
56+
});
57+
58+
test('a missing webOS CLI fails as TOOL_MISSING with a hint naming aresBin', async () => {
59+
const bin = fakeAresBin({});
60+
61+
const listing = createAres(bin, process.env).listDevices();
62+
63+
await assert.rejects(listing, (error: { code?: string; details?: { hint?: string } }) => {
64+
assert.equal(error.code, 'TOOL_MISSING');
65+
assert.match(error.details?.hint ?? '', /aresBin/);
66+
return true;
67+
});
68+
});
69+
70+
test('a failing ares command reports its stderr in a COMMAND_FAILED message', async () => {
71+
const bin = mkdtempForTestSync('ares-');
72+
const script = path.join(bin, 'ares-launch');
73+
writeFileSync(script, `#!/bin/sh\necho 'ERR! Connection timed out' >&2\nexit 1\n`);
74+
chmodSync(script, 0o755);
75+
76+
const launch = createAres(bin, process.env).forDevice('living-room').launch('com.example.app');
77+
78+
await assert.rejects(launch, (error: { code?: string; message?: string }) => {
79+
assert.equal(error.code, 'COMMAND_FAILED');
80+
assert.equal(error.message, 'ares-launch failed: ERR! Connection timed out');
81+
return true;
82+
});
83+
});
84+
85+
test('a cancelled ares-launch keeps its request-canceled reason', async () => {
86+
const bin = mkdtempForTestSync('ares-');
87+
const script = path.join(bin, 'ares-launch');
88+
writeFileSync(script, '#!/bin/sh\nsleep 30\n');
89+
chmodSync(script, 0o755);
90+
const cancel = new AbortController();
91+
92+
const launch = createAres(bin, process.env)
93+
.forDevice('living-room')
94+
.launch('com.example.app', cancel.signal);
95+
cancel.abort();
96+
97+
await assert.rejects(launch, isRequestCanceledError);
98+
});

0 commit comments

Comments
 (0)