Skip to content

Commit a9301f3

Browse files
committed
feat(provider-webos): add LG webOS TV provider plugin
Add @agent-device/webos, an optional provider plugin that drives web apps on LG webOS TVs, and the private @agent-device/tv-cdp package it bundles, which holds the CDP page layer a later Tizen plugin will share. The TV enters as an Android TV device owned by the plugin (id webos:<ares device>). Lifecycle runs through ares-launch, and the page is driven over the Chrome DevTools Protocol through the ares-inspect tunnel. v1 covers open, close, apps, snapshot, remote keys (directions, select, back), pointer, text, scroll, screenshot, logs and appstate. Releasing a lease closes the page and the tunnel, and a page whose socket, tunnel or app goes away is detached, so later commands ask for open again. - Kernel: add the cdp-dom snapshot producer on the android channel so DOM snapshots do not pose as UIAutomator output. - Capture kit: let shapeDesktopSurfaceSnapshot take an interactive test, since every DOM node carries a rect. - Packed-install smoke: let scripts/check-provider-plugin.mjs run fixtures for providers without credentials, and add the webOS fixture. - Docs: add the LG webOS page, mention it in plugins, commands and the sidebar, and add a webOS flow to help remote.
1 parent a0e7c7d commit a9301f3

51 files changed

Lines changed: 3056 additions & 42 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.fallowrc.json‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@
1111
"packages/provider-revyl/test/package-smoke.mjs",
1212
"packages/provider-revyl/tsdown.config.ts",
1313
"packages/provider-revyl/src/plugin.ts",
14+
"packages/provider-webos/test/package-smoke.mjs",
15+
"packages/provider-webos/tsdown.config.ts",
1416
"vitest.mutation.config.ts",
1517
"src/sdk/index.ts",
1618
"src/sdk/io.ts",
@@ -98,6 +100,10 @@
98100
// through this workspace link to core source so typecheck needs no build; Fallow then credits
99101
// the source files rather than the package.
100102
"agent-device",
103+
// Nothing in the root imports the webOS plugin or the tv-cdp package it bundles, but layering
104+
// requires the root manifest to declare every workspace package.
105+
"@agent-device/tv-cdp",
106+
"@agent-device/webos",
101107
// Oxlint resolves the shared config's JS plugins dynamically from their package names.
102108
"@nkzw/eslint-plugin",
103109
"eslint-plugin-no-only-tests",

‎examples/sdk/tsconfig.json‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@
6161
{ "path": "../../packages/provider-limrun" },
6262
{ "path": "../../packages/provider-revyl" },
6363
{ "path": "../../packages/provider-testmu" },
64+
{ "path": "../../packages/provider-webos" },
6465
{ "path": "../../packages/provider-webdriver" },
6566
{ "path": "../../packages/provision-kit" },
6667
{ "path": "../../packages/proxy" },
@@ -71,6 +72,7 @@
7172
{ "path": "../../packages/selectors" },
7273
{ "path": "../../packages/session-journal" },
7374
{ "path": "../../packages/test-kit" },
75+
{ "path": "../../packages/tv-cdp" },
7476
{ "path": "../../packages/xml" }
7577
]
7678
}

‎package.json‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -336,6 +336,7 @@
336336
"@agent-device/selectors": "workspace:*",
337337
"@agent-device/session-journal": "workspace:*",
338338
"@agent-device/test-kit": "workspace:*",
339+
"@agent-device/tv-cdp": "workspace:*",
339340
"@agent-device/xml": "workspace:*",
340341
"@arethetypeswrong/cli": "^0.18.5",
341342
"@chenglou/freerange": "^0.0.4",
@@ -375,6 +376,7 @@
375376
"yaml": "^2.9.0",
376377
"yauzl": "^3.4.0",
377378
"@agent-device/testmu": "workspace:*",
378-
"@agent-device/revyl": "workspace:*"
379+
"@agent-device/revyl": "workspace:*",
380+
"@agent-device/webos": "workspace:*"
379381
}
380382
}

‎packages/capture-kit/src/snapshot-desktop-projection.ts‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,19 +16,19 @@ const INTERACTIVE_ROLE_TOKENS = [
1616
'switch',
1717
] as const;
1818

19+
/** `isInteractive` overrides the default test for producers whose every node carries a rect. */
1920
export function shapeDesktopSurfaceSnapshot(
2021
data: SnapshotResult,
2122
options: Pick<SnapshotOptions, 'depth' | 'interactiveOnly' | 'scope'>,
23+
isInteractive: (node: RawSnapshotNode) => boolean = isInteractiveSnapshotNode,
2224
): SnapshotResult {
2325
let nodes = data.nodes ?? [];
2426
if (options.scope) {
2527
nodes = scopeSnapshotNodes(nodes, options.scope, (range) =>
26-
options.interactiveOnly
27-
? nodes.slice(range.start, range.end).some(isInteractiveSnapshotNode)
28-
: true,
28+
options.interactiveOnly ? nodes.slice(range.start, range.end).some(isInteractive) : true,
2929
);
3030
}
31-
if (options.interactiveOnly) nodes = filterInteractiveSnapshotNodes(nodes);
31+
if (options.interactiveOnly) nodes = filterInteractiveSnapshotNodes(nodes, isInteractive);
3232
if (typeof options.depth === 'number') nodes = filterSnapshotNodesByDepth(nodes, options.depth);
3333
return { ...data, nodes };
3434
}
@@ -46,12 +46,15 @@ export function scopeSnapshotNodes(
4646
return reindexSnapshotNodes(slice, slice[0]?.depth ?? 0);
4747
}
4848

49-
function filterInteractiveSnapshotNodes(nodes: RawSnapshotNode[]): RawSnapshotNode[] {
49+
function filterInteractiveSnapshotNodes(
50+
nodes: RawSnapshotNode[],
51+
isInteractive: (node: RawSnapshotNode) => boolean,
52+
): RawSnapshotNode[] {
5053
if (nodes.length === 0) return nodes;
5154
const byIndex = new Map(nodes.map((node) => [node.index, node]));
5255
const keepIndexes = new Set<number>();
5356
for (const node of nodes) {
54-
if (!isInteractiveSnapshotNode(node)) continue;
57+
if (!isInteractive(node)) continue;
5558
let current: RawSnapshotNode | undefined = node;
5659
while (current) {
5760
if (keepIndexes.has(current.index)) break;

‎packages/capture-kit/src/snapshot-occlusion.test.ts‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -237,6 +237,42 @@ test('annotateSnapshotNodesCoveredByPolicy refuses without guessing a cover', ()
237237
assert.equal(result[0]?.coveredBy, undefined);
238238
});
239239

240+
test('annotateSnapshotNodesCoveredByPolicy publishes the cover a backend hit test names', () => {
241+
const target: RawSnapshotNode = {
242+
index: 0,
243+
type: 'button',
244+
hittable: true,
245+
label: 'Watch',
246+
rect: { x: 136, y: 731, width: 300, height: 72 },
247+
};
248+
const card = {
249+
type: 'button',
250+
label: 'Top story',
251+
rect: { x: 104, y: 208, width: 1114, height: 627 },
252+
};
253+
254+
const [result] = annotateSnapshotNodesCoveredByPolicy([target], () => card);
255+
256+
assert.equal(result?.interactionBlocked, 'covered');
257+
assert.equal(result?.hittable, false);
258+
assert.deepEqual(result?.coveredBy, { kind: 'button', label: 'Top story', rect: card.rect });
259+
});
260+
261+
test('annotateSnapshotNodesCoveredByPolicy names no cover when the hit element is an anonymous box', () => {
262+
const target: RawSnapshotNode = {
263+
index: 0,
264+
type: 'button',
265+
hittable: true,
266+
label: 'Watch',
267+
rect: { x: 136, y: 731, width: 300, height: 72 },
268+
};
269+
270+
const [result] = annotateSnapshotNodesCoveredByPolicy([target], () => ({ type: 'group' }));
271+
272+
assert.equal(result?.interactionBlocked, 'covered');
273+
assert.equal(result?.coveredBy, undefined);
274+
});
275+
240276
test('annotateCoveredSnapshotNodes leaves an uncovered touch target unchanged', () => {
241277
const target: RawSnapshotNode = {
242278
index: 0,

‎packages/capture-kit/src/snapshot-occlusion.ts‎

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -152,7 +152,7 @@ const ANONYMOUS_CONTAINER_KINDS = new Set(['element', 'group', 'other']);
152152
* no cover, and the refusal keeps its generic recovery. A named kind (`dialog`, `tab-bar`, `sheet`)
153153
* still describes itself with no label to lean on.
154154
*/
155-
function describeCoveringNode(node: RawSnapshotNode): SnapshotCoveringNode | null {
155+
function describeCoveringNode(node: CoveringNodeEvidence): SnapshotCoveringNode | null {
156156
const label = node.label?.trim();
157157
const kind = formatRole(node.type ?? 'Element');
158158
if (!label && ANONYMOUS_CONTAINER_KINDS.has(kind)) return null;
@@ -164,14 +164,24 @@ function describeCoveringNode(node: RawSnapshotNode): SnapshotCoveringNode | nul
164164
};
165165
}
166166

167-
/** Applies backend-owned covered decisions through the same publication annotation contract. */
167+
/** What a backend hit test observed on top of a node, described like a scanned cover. */
168+
export type CoveringNodeEvidence = Pick<RawSnapshotNode, 'type' | 'label' | 'rect'>;
169+
170+
/**
171+
* Applies backend-owned covered decisions through the same publication annotation contract. A
172+
* decision that carries the covering element publishes its description; `true` refuses without one.
173+
*/
168174
export function annotateSnapshotNodesCoveredByPolicy(
169175
nodes: RawSnapshotNode[],
170-
isCovered: (node: RawSnapshotNode) => boolean,
176+
isCovered: (node: RawSnapshotNode) => boolean | CoveringNodeEvidence,
171177
): RawSnapshotNode[] {
172-
const coveredPositions: Array<{ position: number; coveredBy?: undefined }> = [];
178+
const coveredPositions: Array<{ position: number; coveredBy?: SnapshotCoveringNode }> = [];
173179
for (const [position, node] of nodes.entries()) {
174-
if (isCandidateTouchNode(node) && isCovered(node)) coveredPositions.push({ position });
180+
if (!isCandidateTouchNode(node)) continue;
181+
const decision = isCovered(node);
182+
if (!decision) continue;
183+
const coveredBy = decision === true ? null : describeCoveringNode(decision);
184+
coveredPositions.push(coveredBy ? { position, coveredBy } : { position });
175185
}
176186
return annotateCoveredPositions(nodes, coveredPositions);
177187
}

‎packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@ const DEFAULT_QUIET_MS_BY_PRODUCER = {
6767
'limrun-ios-tree': DEFAULT_STABLE_QUIET_MS,
6868
'revyl-ios-tree': DEFAULT_STABLE_QUIET_MS,
6969
'android-uiautomator': DEFAULT_STABLE_QUIET_MS,
70+
'cdp-dom': DEFAULT_STABLE_QUIET_MS,
7071
'harmonyos-uitest': DEFAULT_STABLE_QUIET_MS,
7172
'macos-helper': DEFAULT_STABLE_QUIET_MS,
7273
'linux-atspi': DEFAULT_STABLE_QUIET_MS,

‎packages/kernel/src/snapshot.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -587,7 +587,7 @@ export type SnapshotProvenance =
587587
| 'limrun-ios-tree'
588588
| 'revyl-ios-tree';
589589
}
590-
| { backend: 'android'; producer: 'android-uiautomator' | 'appium-source' }
590+
| { backend: 'android'; producer: 'android-uiautomator' | 'appium-source' | 'cdp-dom' }
591591
| { backend: 'harmonyos-arkui'; producer: 'harmonyos-uitest' }
592592
| { backend: 'macos-helper'; producer: 'macos-helper' }
593593
| { backend: 'linux-atspi'; producer: 'linux-atspi' }

‎packages/provider-testmu/test/package-smoke.mjs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
export const provider = 'testmu';
12
export const args = [
23
'--platform',
34
'android',

‎packages/provider-webos/README.md‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
# @agent-device/webos
2+
3+
Drive web apps on LG webOS TVs with [agent-device](https://agent-device.dev). The plugin launches
4+
the app with the webOS CLI, opens the `ares-inspect` tunnel, and drives the page over the Chrome
5+
DevTools Protocol: remote keys, pointer, text, snapshots, screenshots and console logs.
6+
7+
Status: experimental. Verified on an LG 32LQ63006LA (webOS TV 22, dev build of a React Native Web
8+
app): `connect`, `open`, `snapshot -i`, `snapshot -s`, `tv-remote` (directions, select, back),
9+
`is focused`, `press @ref --settle`, `wait text`, `find`, `screenshot`, `logs`, `apps`,
10+
`appstate`, `close`, `disconnect`.
11+
12+
## How it plugs in
13+
14+
agent-device leases only iOS, Android and HarmonyOS devices, so a webOS TV enters as an
15+
**Android TV device that this plugin owns** (`platform: android`, `target: tv`, id
16+
`webos:<ares device>`). Every command on that device goes to this plugin's runtime, never to adb.
17+
18+
## Requirements
19+
20+
- webOS CLI (`ares-*`) on `PATH`, or the directory that holds it passed as `aresBin`; see LG's
21+
[webOS CLI installation](https://webostv.developer.lge.com/develop/tools/cli-installation)
22+
- The TV in Developer Mode and registered with `ares-setup-device`; see LG's
23+
[App Testing with Developer Mode App](https://webostv.developer.lge.com/develop/getting-started/developer-mode-app)
24+
- The app installed on the TV
25+
- The TV and this host on the same network
26+
27+
## Install
28+
29+
```sh
30+
agent-device plugins add @agent-device/webos
31+
agent-device daemon stop # the next command starts a daemon with the plugin loaded
32+
```
33+
34+
Plugin options: `aresBin` (directory with the `ares-*` CLIs) and `device` (default ares device
35+
name).
36+
37+
To try a local build, pack the package and install the tarball into a plugin store installation:
38+
39+
```sh
40+
pnpm --filter @agent-device/webos pack --out /tmp/agent-device-webos.tgz
41+
installation=$(uuidgen)
42+
npm install --prefix ~/.agent-device/plugins/$installation /tmp/agent-device-webos.tgz
43+
```
44+
45+
Then register `{"plugins": {"@agent-device/webos": {"installation": "<installation>"}}}` in
46+
`~/.agent-device/config.json` and run `agent-device daemon stop`.
47+
48+
## Use
49+
50+
```sh
51+
agent-device daemon stop # the lease TTL is read when the daemon starts
52+
export AGENT_DEVICE_LEASE_TTL_MS=1800000 AGENT_DEVICE_LEASE_MAX_TTL_MS=1800000 # see Known limitations
53+
agent-device connect webos [--device <ares device>] # defaults to the ares default device
54+
agent-device open <appId> # ares-launch, then attach over CDP
55+
agent-device tv-remote press down
56+
agent-device snapshot -i
57+
agent-device screenshot ./tv.png
58+
agent-device logs start # then logs path
59+
agent-device close <appId>
60+
agent-device disconnect
61+
```
62+
63+
Later commands use the session `connect` created; `connect` prints its name for `--session`.
64+
65+
## Supported
66+
67+
| Command | How |
68+
| --- | --- |
69+
| `open` / `close` | `ares-launch` / `ares-launch --close`, then the `ares-inspect` tunnel |
70+
| `tv-remote press up/down/left/right/select/back`, `back` | `Input.dispatchKeyEvent` with the LG remote codes (Back = 461) |
71+
| `appstate` | the attached app while its page is visible |
72+
| `apps` | `ares-install --listfull` |
73+
| `logs` | console calls, exceptions and browser log entries of the attached page, one line each |
74+
| `snapshot`, `find`, `is`, `get`, `wait` | DOM walk over `Runtime.evaluate`: visible interactive, labelled or text nodes, focus from `document.activeElement`; `-s`, `-d` and `-i` apply to the walk. A control whose centre hit-tests to another element is marked `[covered]` |
75+
| `screenshot` | `Page.captureScreenshot` |
76+
| `click`, `press <x> <y>`, `longpress`, `hover` | `Input.dispatchMouseEvent` (Magic Remote pointer), ending each move with a small wiggle that wakes the cursor |
77+
| `fill` | pointer press on the field, select its contents, then `Input.insertText`, or Backspace for an empty value |
78+
| `type` | `Input.insertText` into the focused element |
79+
| `scroll` | `mouseWheel` at the centre of the scroll container or viewport, by the requested distance |
80+
81+
Not yet: system buttons (`home`, menu, volume, channel and media keys), `network dump`, `perf`,
82+
`install` and `uninstall`, recording, URLs and deep links in `open`, `power`.
83+
84+
## Known limitations
85+
86+
- **Idle leases expire after 60 s by default, and expiry ends the session.** agent-device gives a
87+
direct-device provider lease its default TTL and beats it only while a command runs. When the
88+
lease ends, by `disconnect` or by expiry, the plugin closes the page connection and the
89+
`ares-inspect` tunnel. After a longer pause, the next command fails with `Lease is not active`
90+
or `No active session` and needs a new `connect` and `open`. To avoid it, run
91+
`agent-device daemon stop`, export `AGENT_DEVICE_LEASE_TTL_MS` and
92+
`AGENT_DEVICE_LEASE_MAX_TTL_MS` (for example `1800000`, 30 min), and run the next command in
93+
that shell. That command starts the daemon with both values.
94+
- **`screenshot` captures the app page, not the TV screen.** When the app is in the background,
95+
for example after Home on the remote, the page is hidden and renders no frames, so `screenshot`
96+
fails at once; `open <appId>` brings the app back. A whole-screen capture is not reachable on a retail TV: SSAP
97+
`tv/executeOneShot` answers `401 insufficient permissions` to unsigned clients, LG rejects the
98+
public signed manifest (`blacklisted certificate`), and the dev-mode shell (`prisoner`) gets no
99+
answer from `luna-send-pub`.
100+
- **Console messages from before the attach are replayed without object previews**, so objects
101+
in them show as `Object`. Messages logged after the attach show their properties.
102+
- **`open` on a running app can reload it** (webOS relaunch), so the first read may hit a document
103+
without a body. `open` waits for a loaded document, and a snapshot of an unloaded page is empty
104+
rather than an error. `open --relaunch` closes the app first, for a cold start.
105+
- **A second DevTools client on the same page hung in testing.** Close Chrome DevTools and other
106+
inspector clients attached to the app before a session.
107+
- **Snapshots are DOM-derived.** Labels come from `aria-label`, `alt`, `title` or own text;
108+
rects are CSS px, which match screenshot pixels only at `devicePixelRatio` 1 (1080p apps).
109+
The walk does not enter iframes or shadow roots, so overlays rendered there (for example a
110+
dev-server error overlay) appear in screenshots but not in snapshots.
111+
- **`click`, `press`, `fill` and `focus` refuse a control marked `[covered]`.** A tap at its
112+
centre would land on the element on top, such as a card's own pressable area. `snapshot -i`
113+
still lists it, and `tv-remote` reaches it through focus.
114+
- **System overlays are outside the page.** The LG menu, on-screen keyboard and other system
115+
layers never appear in `screenshot` or `snapshot`.

0 commit comments

Comments
 (0)