Skip to content

Commit 50e9dcd

Browse files
committed
feat(provider-tizen): add Samsung Tizen TV provider plugin
Add @agent-device/tizen, an optional provider plugin that drives web apps on Samsung Tizen TVs, built on the shared tv-cdp layer. The TV enters as an Android TV device owned by the plugin (id tizen:<device>). open kills the app, debug-launches it with sdb shell 0 debug, forwards the DevTools port with sdb forward, and drives the page over the Chrome DevTools Protocol. - sdb is found in the VS Code Tizen extension's SDK ($TIZEN_SDK or ~/.tizen.path.config) or at the sdbPath option; --device takes a TV IP (sdb connect <ip>:26101 runs when needed) or an sdb serial. - v1 scope matches webOS: connect, open (--relaunch), close, apps (vd_applist), appstate, snapshot, keys (directions, select, back as XF86Back), pointer, text, scroll, screenshot and logs. - Docs: add the Samsung Tizen page, mention it in plugins, commands and the sidebar, and add a Tizen flow to help remote.
1 parent b942ccd commit 50e9dcd

26 files changed

Lines changed: 877 additions & 4 deletions

‎.fallowrc.json‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@
1313
"packages/provider-revyl/src/plugin.ts",
1414
"packages/provider-webos/test/package-smoke.mjs",
1515
"packages/provider-webos/tsdown.config.ts",
16+
"packages/provider-tizen/test/package-smoke.mjs",
17+
"packages/provider-tizen/tsdown.config.ts",
1618
"vitest.mutation.config.ts",
1719
"src/sdk/index.ts",
1820
"src/sdk/io.ts",
@@ -100,8 +102,9 @@
100102
// through this workspace link to core source so typecheck needs no build; Fallow then credits
101103
// the source files rather than the package.
102104
"agent-device",
103-
// Nothing in the root imports the webOS plugin or the tv-cdp package it bundles, but layering
105+
// Nothing in the root imports the TV plugins or the tv-cdp package they bundle, but layering
104106
// requires the root manifest to declare every workspace package.
107+
"@agent-device/tizen",
105108
"@agent-device/tv-cdp",
106109
"@agent-device/webos",
107110
// Oxlint resolves the shared config's JS plugins dynamically from their package names.

‎examples/sdk/tsconfig.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@
6262
{ "path": "../../packages/provider-revyl" },
6363
{ "path": "../../packages/provider-testmu" },
6464
{ "path": "../../packages/provider-webos" },
65+
{ "path": "../../packages/provider-tizen" },
6566
{ "path": "../../packages/provider-webdriver" },
6667
{ "path": "../../packages/provision-kit" },
6768
{ "path": "../../packages/proxy" },

‎package.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -377,6 +377,7 @@
377377
"yauzl": "^3.4.0",
378378
"@agent-device/testmu": "workspace:*",
379379
"@agent-device/revyl": "workspace:*",
380-
"@agent-device/webos": "workspace:*"
380+
"@agent-device/webos": "workspace:*",
381+
"@agent-device/tizen": "workspace:*"
381382
}
382383
}

‎packages/provider-tizen/README.md‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
# @agent-device/tizen
2+
3+
Drive web apps on Samsung Tizen TVs with [agent-device](https://agent-device.dev). The plugin
4+
debug-launches the app over `sdb`, forwards its DevTools port, and drives the page over the Chrome
5+
DevTools Protocol: remote keys, pointer, text, snapshots, screenshots and console logs.
6+
7+
Status: experimental. Verified on a Samsung Smart Monitor M7 (Tizen 9.0, packaged build of a React
8+
Native Web app): `connect`, `open`, `open --relaunch`, `snapshot -i`, `tv-remote` (directions,
9+
select, back), `back`, `is focused`, `press`, `find ... click`, `hover`, `longpress`, `fill`,
10+
`type`, `wait`, `scroll`, `screenshot`, `logs`, `apps`, `appstate`, `close`, `disconnect`.
11+
12+
## How it plugs in
13+
14+
agent-device leases only iOS, Android and HarmonyOS devices, so a Tizen TV enters as an
15+
**Android TV device that this plugin owns** (`platform: android`, `target: tv`, id
16+
`tizen:<device>`). Every command on that device goes to this plugin's runtime, never to adb.
17+
18+
## Requirements
19+
20+
- `sdb` from the Tizen SDK of the VS Code Tizen extension, set up with its Set Up step. The plugin
21+
finds it in the SDK that `$TIZEN_SDK` or `~/.tizen.path.config` names, or at the `sdbPath`
22+
option; it is not on `PATH`
23+
- The TV in Developer Mode with this host's IP set as the developer IP; see Samsung's
24+
[Connecting the TV and SDK](https://developer.samsung.com/smarttv/develop/getting-started/using-sdk/tv-device.html)
25+
- The app installed on the TV from a debug-enabled package
26+
- The TV and this host on the same network
27+
28+
## Install
29+
30+
```sh
31+
agent-device plugins add @agent-device/tizen
32+
agent-device daemon stop # the next command starts a daemon with the plugin loaded
33+
```
34+
35+
Plugin options: `sdbPath` (the `sdb` binary, when the SDK is not found) and `device` (default TV
36+
IP or sdb serial).
37+
38+
To try a local build, pack the package and install the tarball into a plugin store installation:
39+
40+
```sh
41+
pnpm --filter @agent-device/tizen pack --out /tmp/agent-device-tizen.tgz
42+
installation=$(uuidgen)
43+
npm install --prefix ~/.agent-device/plugins/$installation /tmp/agent-device-tizen.tgz
44+
```
45+
46+
Then register `{"plugins": {"@agent-device/tizen": {"installation": "<installation>"}}}` in
47+
`~/.agent-device/config.json` and run `agent-device daemon stop`.
48+
49+
## Use
50+
51+
```sh
52+
agent-device daemon stop # the lease TTL is read when the daemon starts
53+
export AGENT_DEVICE_LEASE_TTL_MS=1800000 AGENT_DEVICE_LEASE_MAX_TTL_MS=1800000 # see Known limitations
54+
agent-device connect tizen --device 192.0.2.10 # TV IP or sdb serial
55+
agent-device open <appId> # kill, debug launch, forward, attach over CDP
56+
agent-device wait text <text on the first screen>
57+
agent-device tv-remote press down
58+
agent-device snapshot -i
59+
agent-device screenshot ./tv.png
60+
agent-device logs start # then logs path
61+
agent-device close <appId>
62+
agent-device disconnect
63+
```
64+
65+
`--device` takes the TV IP (`sdb connect <ip>:26101` runs when it is not connected yet) or an sdb
66+
serial. Without it, the plugin uses the `device` option, then the only TV connected to `sdb`.
67+
`connect` does not contact the TV; `open` connects it. Later commands use the session `connect`
68+
created; `connect` prints its name for `--session`.
69+
70+
## Supported
71+
72+
| Command | How |
73+
| --- | --- |
74+
| `open` / `close` | `sdb shell 0 was_kill`, `sdb shell 0 debug` (prints the DevTools port), `sdb forward`, then CDP / `sdb shell 0 was_kill` |
75+
| `tv-remote press up/down/left/right/select/back`, `back` | `Input.dispatchKeyEvent` with the Samsung remote codes (Back = `XF86Back`, 10009) |
76+
| `appstate` | the attached app while its page is visible |
77+
| `apps` | `sdb shell 0 vd_applist`: user-installed apps with their titles |
78+
| `logs` | console calls, exceptions and browser log entries of the attached page, one line each |
79+
| `snapshot`, `find`, `is`, `get`, `wait` | DOM walk over `Runtime.evaluate`: visible interactive, labelled or text nodes, focus from `document.activeElement`; `-s`, `-d` and `-i` apply to the walk |
80+
| `screenshot` | `Page.captureScreenshot` |
81+
| `click`, `press <x> <y>`, `longpress`, `hover` | `Input.dispatchMouseEvent` |
82+
| `fill` | pointer press on the field, select its contents, then `Input.insertText` |
83+
| `type` | `Input.insertText` into the focused element |
84+
| `scroll` | `mouseWheel` at the centre of the scroll container or viewport, by the requested distance |
85+
86+
Not yet: system buttons (`home`, menu, volume, channel and media keys), `network dump`, `perf`,
87+
`install` and `uninstall`, recording, URLs and deep links in `open`, `power`.
88+
89+
## Known limitations
90+
91+
- **`open` is always a cold start.** `sdb shell 0 debug` launches the app and cannot attach to a
92+
running one, so `open` kills the app first. `open` returns once the document has loaded, which
93+
can be before the app leaves its splash screen; wait for app content (`wait text ...`) before
94+
the first snapshot.
95+
- **Idle leases expire after 60 s by default, and expiry ends the session.** agent-device gives a
96+
direct-device provider lease its default TTL and beats it only while a command runs. When the
97+
lease ends, by `disconnect` or by expiry, the plugin closes the page connection and removes the
98+
`sdb` forward. After a longer pause, the next command fails with `Lease is not active` or
99+
`No active session` and needs a new `connect` and `open`. To avoid it, run
100+
`agent-device daemon stop`, then start the daemon with `AGENT_DEVICE_LEASE_TTL_MS` and
101+
`AGENT_DEVICE_LEASE_MAX_TTL_MS` set (for example 30 min).
102+
- **`screenshot` captures the app page, not the TV screen.** The restricted `sdb shell 0` has no
103+
screen-capture command. When the app is in the background, the page is hidden and renders no
104+
frames, so `screenshot` fails at once; `open <appId>` brings the app back. While the on-screen
105+
keyboard is open, the page viewport and so the screenshot are shorter.
106+
- **`appstate` knows only the attached app.** The TV's REST API (`:8001/api/v2/applications/<id>`)
107+
reports `visible: false` for an app in front of the user, and `sdb shell 0` has no
108+
foreground-app command.
109+
- **Console messages from before the attach are replayed without object previews**, so objects
110+
in them show as `Object`. Messages logged after the attach show their properties.
111+
- **A second DevTools client on the same page can hang the session.** Close Chrome DevTools and
112+
other inspector clients attached to the app before a session.
113+
- **Snapshots are DOM-derived.** Labels come from `aria-label`, `alt`, `title` or own text;
114+
rects are CSS px, which match screenshot pixels only at `devicePixelRatio` 1 (1080p apps).
115+
The walk does not enter iframes or shadow roots. `is text` on an input compares its label, not
116+
its value; the value is in `snapshot --raw`.
117+
- **System overlays are outside the page.** The Samsung menu, on-screen keyboard and other system
118+
layers never appear in `screenshot` or `snapshot`.
Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
{
2+
"name": "@agent-device/tizen",
3+
"version": "0.21.36-dev",
4+
"type": "module",
5+
"description": "Samsung Tizen TV web app plugin for agent-device.",
6+
"files": [
7+
"dist"
8+
],
9+
"scripts": {
10+
"build": "tsdown --config tsdown.config.ts",
11+
"prepack": "pnpm build"
12+
},
13+
"exports": {
14+
".": {
15+
"types": "./src/index.ts",
16+
"default": "./src/index.ts"
17+
},
18+
"./agent-device": {
19+
"types": "./src/index.ts",
20+
"default": "./src/index.ts"
21+
},
22+
"./package.json": "./package.json"
23+
},
24+
"devDependencies": {
25+
"@agent-device/host-kit": "workspace:*",
26+
"@agent-device/kernel": "workspace:*",
27+
"@agent-device/test-kit": "workspace:*",
28+
"@agent-device/tv-cdp": "workspace:*",
29+
"agent-device": "workspace:*"
30+
},
31+
"publishConfig": {
32+
"exports": {
33+
".": {
34+
"import": "./dist/agent-device.mjs"
35+
},
36+
"./agent-device": {
37+
"import": "./dist/agent-device.mjs"
38+
}
39+
},
40+
"access": "public"
41+
},
42+
"license": "MIT",
43+
"repository": {
44+
"type": "git",
45+
"url": "git+https://github.com/callstack/agent-device.git",
46+
"directory": "packages/provider-tizen"
47+
},
48+
"homepage": "https://agent-device.dev/docs/tizen"
49+
}
Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
import type { AgentDevicePlugin } from 'agent-device/plugins';
2+
import { tvPluginConnection } from '@agent-device/tv-cdp/connection';
3+
4+
import { PROVIDER, type TizenOptions } from './provider.ts';
5+
6+
function tizen(options: TizenOptions = {}) {
7+
return {
8+
apiVersion: 2,
9+
provider: PROVIDER,
10+
credentials: { variables: [] },
11+
connection: tvPluginConnection({
12+
provider: PROVIDER,
13+
service: 'Samsung Tizen',
14+
verificationMessage: 'Tizen TVs are reached over sdb when a session opens.',
15+
defaultDevice: options.device,
16+
}),
17+
setup: async (host) => (await import('./runtime.ts')).createTizenRuntime(options, host),
18+
} satisfies AgentDevicePlugin;
19+
}
20+
21+
export { tizen as default };
Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
import { NAVIGATION_KEYS, type RemoteKeys } from '@agent-device/tv-cdp';
2+
3+
export const TIZEN_REMOTE_KEYS: RemoteKeys = {
4+
...NAVIGATION_KEYS,
5+
back: { key: 'XF86Back', code: 'XF86Back', keyCode: 10009 },
6+
};
Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
export const PROVIDER = 'tizen';
2+
3+
export type TizenOptions = Readonly<{
4+
/** Path to `sdb` when it is not found in the Tizen SDK named by $TIZEN_SDK or ~/.tizen.path.config. */
5+
sdbPath?: string;
6+
/** TV to drive when the lease names none: an sdb serial or an IP; defaults to the only connected TV. */
7+
device?: string;
8+
}>;
Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
import assert from 'node:assert/strict';
2+
import { chmodSync, readFileSync, writeFileSync } from 'node:fs';
3+
import path from 'node:path';
4+
import { AppError } from '@agent-device/kernel/errors';
5+
import { mkdtempForTestSync } from '@agent-device/test-kit';
6+
import { test } from 'vitest';
7+
8+
import { createTizenRuntime } from './runtime.ts';
9+
10+
// An sdb whose `devices` lists the serials in a state file, and whose `connect` adds one.
11+
function fakeSdb(connected: readonly string[]) {
12+
const dir = mkdtempForTestSync('sdb-');
13+
const state = path.join(dir, 'connected');
14+
const calls = path.join(dir, 'calls');
15+
writeFileSync(state, connected.map((serial) => `${serial}\n`).join(''));
16+
writeFileSync(calls, '');
17+
const sdb = path.join(dir, 'sdb');
18+
writeFileSync(
19+
sdb,
20+
[
21+
'#!/bin/sh',
22+
`echo "$*" >> '${calls}'`,
23+
'case "$1" in',
24+
` devices) echo 'List of devices attached'; while read -r serial; do printf '%s\\tdevice\\tQE55Q80\\n' "$serial"; done < '${state}' ;;`,
25+
` connect) echo "$2" >> '${state}'; echo "connected to $2" ;;`,
26+
'esac',
27+
].join('\n'),
28+
);
29+
chmodSync(sdb, 0o755);
30+
return { sdb, calls: () => readFileSync(calls, 'utf8').trim().split('\n') };
31+
}
32+
33+
const host = {
34+
env: process.env,
35+
createError: (code: string, message: string, details?: Record<string, unknown>) =>
36+
new AppError(code as ConstructorParameters<typeof AppError>[0], message, details),
37+
};
38+
39+
const lease = (leaseId: string) => ({
40+
leaseId,
41+
tenantId: 'tenant',
42+
runId: 'run',
43+
backend: 'android-instance' as const,
44+
leaseProvider: 'tizen',
45+
createdAt: 0,
46+
heartbeatAt: 0,
47+
expiresAt: Number.MAX_SAFE_INTEGER,
48+
});
49+
50+
test('a TV named by IP is connected over sdb and keeps that name as a Tizen-owned Android TV', async () => {
51+
const fake = fakeSdb([]);
52+
const { runtime } = await createTizenRuntime({ sdbPath: fake.sdb }, host);
53+
54+
const allocated = await runtime.leaseLifecycle.allocate(lease('one'), {
55+
flags: { device: '192.0.2.10' },
56+
});
57+
58+
assert.deepEqual(fake.calls(), ['devices', 'connect 192.0.2.10:26101', 'devices']);
59+
assert.equal(allocated?.device.name, '192.0.2.10');
60+
assert.equal(allocated?.device.id, 'tizen:192.0.2.10');
61+
assert.equal(allocated?.device.platform, 'android');
62+
assert.equal(allocated?.device.target, 'tv');
63+
});
64+
65+
test('without --device, the only connected TV is used and no connect runs', async () => {
66+
const fake = fakeSdb(['192.0.2.10:26101']);
67+
const { runtime } = await createTizenRuntime({ sdbPath: fake.sdb }, host);
68+
69+
const allocated = await runtime.leaseLifecycle.allocate(lease('one'));
70+
71+
assert.equal(allocated?.device.id, 'tizen:192.0.2.10:26101');
72+
assert.deepEqual(fake.calls(), ['devices']);
73+
});
74+
75+
test('without --device and with two TVs connected, allocation asks for one', async () => {
76+
const fake = fakeSdb(['192.0.2.10:26101', '192.0.2.11:26101']);
77+
const { runtime } = await createTizenRuntime({ sdbPath: fake.sdb }, host);
78+
79+
const allocation = runtime.leaseLifecycle.allocate(lease('one'));
80+
81+
await assert.rejects(allocation, (error: AppError) => {
82+
assert.equal(error.code, 'DEVICE_NOT_FOUND');
83+
assert.deepEqual(error.details?.available, ['192.0.2.10:26101', '192.0.2.11:26101']);
84+
return true;
85+
});
86+
});
Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
import { createTvRuntime, type TvConnection, type TvRuntimeHost } from '@agent-device/tv-cdp';
2+
3+
import { TIZEN_REMOTE_KEYS } from './keys.ts';
4+
import { PROVIDER, type TizenOptions } from './provider.ts';
5+
import { createSdb, sdbSerial } from './sdb.ts';
6+
import { locateSdb, type HostEnv } from './tools.ts';
7+
8+
type TizenRuntimeHost = TvRuntimeHost & Readonly<{ env: HostEnv }>;
9+
10+
async function readySerials(sdb: ReturnType<typeof createSdb>) {
11+
return (await sdb.listDevices())
12+
.filter((device) => device.state === 'device')
13+
.map((device) => device.serial);
14+
}
15+
16+
export function createTizenRuntime(options: TizenOptions, host: TizenRuntimeHost) {
17+
const sdb = createSdb(() => locateSdb(options.sdbPath, host.env), host.env);
18+
19+
async function resolveSerial(requested: string | undefined) {
20+
const connected = await readySerials(sdb);
21+
if (!requested) {
22+
if (connected.length === 1) return connected[0]!;
23+
throw host.createError(
24+
'DEVICE_NOT_FOUND',
25+
connected.length === 0
26+
? 'No Tizen TV is connected to sdb'
27+
: 'More than one Tizen TV is connected to sdb',
28+
{ available: connected, hint: 'Pass --device <TV IP or sdb serial>.' },
29+
);
30+
}
31+
const serial = sdbSerial(requested);
32+
if (connected.includes(serial)) return serial;
33+
await sdb.connect(serial);
34+
if ((await readySerials(sdb)).includes(serial)) return serial;
35+
throw host.createError('DEVICE_NOT_FOUND', `sdb could not connect to the Tizen TV ${serial}`, {
36+
available: connected,
37+
hint: 'Check that the TV is on and in Developer Mode with this host IP set as the developer IP.',
38+
});
39+
}
40+
41+
async function connectTv(flagDevice: string | undefined): Promise<TvConnection> {
42+
const requested = flagDevice ?? options.device;
43+
const tv = sdb.forDevice(await resolveSerial(requested));
44+
return {
45+
// The daemon selects the leased device by the name --device gave, so the TV keeps it.
46+
name: requested ?? tv.serial,
47+
launch: async (appId) => await tv.launch(appId),
48+
close: tv.kill,
49+
listApps: tv.listApps,
50+
};
51+
}
52+
53+
return createTvRuntime(
54+
{ provider: PROVIDER, label: 'Tizen', remoteKeys: TIZEN_REMOTE_KEYS, connectTv },
55+
host,
56+
);
57+
}

0 commit comments

Comments
 (0)