|
| 1 | +import fs from 'node:fs'; |
| 2 | +import os from 'node:os'; |
| 3 | +import path from 'node:path'; |
| 4 | +import assert from 'node:assert/strict'; |
| 5 | +import test from 'node:test'; |
| 6 | +import { execFileSync } from 'node:child_process'; |
| 7 | +import { fileURLToPath } from 'node:url'; |
| 8 | + |
| 9 | +// Compiles every fenced ```ts snippet in website/docs/docs/client-api.md against |
| 10 | +// agent-device's real `agent-device/*` subpath sources, so a doc snippet that no |
| 11 | +// longer compiles — wrong property, renamed export, changed signature — fails |
| 12 | +// CI instead of only being caught if an examples/sdk/*.ts file happens to drift |
| 13 | +// the same way. src/__tests__/client-api-examples-drift.test.ts covers the |
| 14 | +// doc's bullet-list API manifest against examples/sdk/*.ts; this is the |
| 15 | +// complementary check the reviewer asked for on #1463's drift guard (checking |
| 16 | +// the doc's actual code, not just its symbol-name manifest). |
| 17 | +// |
| 18 | +// Lives in the Node integration lane, not vitest's unit-core: it spawns a real |
| 19 | +// tsc Program over a chunk of src/, which comfortably exceeds the unit suite's |
| 20 | +// 2.5s slow-test budget (see docs/agents/testing.md). |
| 21 | + |
| 22 | +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..'); |
| 23 | +const CLIENT_API_DOC_PATH = path.join(repoRoot, 'website/docs/docs/client-api.md'); |
| 24 | +const EXAMPLES_SDK_TSCONFIG = path.join(repoRoot, 'examples/sdk/tsconfig.json'); |
| 25 | +const TSC_BIN = path.join(repoRoot, 'node_modules/.bin/tsc'); |
| 26 | + |
| 27 | +// Extracts every fenced ```ts code block from the doc, in document order. Each |
| 28 | +// block is compiled standalone. |
| 29 | +function extractTsSnippets(markdown: string): string[] { |
| 30 | + return [...markdown.matchAll(/```ts\n([\s\S]*?)```/g)].map((match) => match[1] ?? ''); |
| 31 | +} |
| 32 | + |
| 33 | +// examples/sdk/tsconfig.json's `paths` is the existing, CI-proven mechanism that |
| 34 | +// resolves `agent-device/...` specifiers straight to `src/sdk/*.ts` so examples |
| 35 | +// typecheck without a build (see that file's own comment). `tsc --showConfig` |
| 36 | +// returns it fully resolved and comment-free, so this reads the one true copy of |
| 37 | +// the subpath map instead of keeping a second, driftable one here. |
| 38 | +function resolveExamplesSdkPaths(): Record<string, string[]> { |
| 39 | + const raw = execFileSync(TSC_BIN, ['-p', EXAMPLES_SDK_TSCONFIG, '--showConfig'], { |
| 40 | + encoding: 'utf8', |
| 41 | + }); |
| 42 | + const configDir = path.dirname(EXAMPLES_SDK_TSCONFIG); |
| 43 | + const paths: Record<string, string[]> = {}; |
| 44 | + for (const [specifier, targets] of Object.entries( |
| 45 | + JSON.parse(raw).compilerOptions.paths as Record<string, string[]>, |
| 46 | + )) { |
| 47 | + paths[specifier] = targets.map((target) => path.resolve(configDir, target)); |
| 48 | + } |
| 49 | + return paths; |
| 50 | +} |
| 51 | + |
| 52 | +// Free identifiers a snippet references without declaring — continuing a |
| 53 | +// `client`/`snapshot` from an earlier snippet in the doc's prose. Typed against |
| 54 | +// the real SDK return types (not `any`) so continuation snippets still get |
| 55 | +// meaningful checking; anything else (illustrative host-glue names the doc |
| 56 | +// invents, like a bridge's own transport function) falls back to `any`, which |
| 57 | +// only weakens the check for that invented name, not for the real SDK calls |
| 58 | +// around it. |
| 59 | +const KNOWN_CONTINUATION_STUB_TYPES: Record<string, string> = { |
| 60 | + client: `ReturnType<typeof import('agent-device').createAgentDeviceClient>`, |
| 61 | + androidClient: `ReturnType<typeof import('agent-device').createAgentDeviceClient>`, |
| 62 | + snapshot: `Awaited<ReturnType<ReturnType<typeof import('agent-device').createAgentDeviceClient>['capture']['snapshot']>>`, |
| 63 | +}; |
| 64 | + |
| 65 | +// Pre-existing gaps this check surfaced that are out of scope for #1463 to fix |
| 66 | +// (they predate this PR and touch unrelated public API surface, not the new |
| 67 | +// examples): flagged for the maintainer rather than silently patched. Remove |
| 68 | +// an entry once the underlying gap is closed. |
| 69 | +const KNOWN_DOC_GAPS = [ |
| 70 | + // "Remote Metro helpers": prepareRemoteMetro/reloadRemoteMetro/stopMetroTunnel |
| 71 | + // aren't exported from `agent-device/metro`, and resolveRemoteConfigProfile |
| 72 | + // isn't exported from `agent-device/remote-config` — the doc documents a |
| 73 | + // public surface neither subpath's src/sdk/*.ts re-exports today. |
| 74 | + `Module '"agent-device/metro"' has no exported member 'prepareRemoteMetro'`, |
| 75 | + `Module '"agent-device/metro"' has no exported member 'reloadRemoteMetro'`, |
| 76 | + `Module '"agent-device/metro"' has no exported member 'stopMetroTunnel'`, |
| 77 | + `'"agent-device/remote-config"' has no exported member named 'resolveRemoteConfigProfile'`, |
| 78 | + // "Web sessions" / audio probe: NetworkOptions/AudioOptions have no `platform` |
| 79 | + // field, even though the CLI's `network`/`audio` commands accept `--platform` |
| 80 | + // — the typed client's contracts are narrower than the CLI surface here. |
| 81 | + `'platform' does not exist in type 'NetworkOptions'`, |
| 82 | + `'platform' does not exist in type 'AudioOptions'`, |
| 83 | +]; |
| 84 | + |
| 85 | +function writeSnippetProgram(snippets: string[]): string { |
| 86 | + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'client-api-doc-snippets-')); |
| 87 | + fs.writeFileSync(path.join(tmpDir, 'package.json'), '{"type":"module"}\n'); |
| 88 | + fs.writeFileSync( |
| 89 | + path.join(tmpDir, 'tsconfig.json'), |
| 90 | + JSON.stringify({ |
| 91 | + extends: path.join(repoRoot, 'tsconfig.json'), |
| 92 | + compilerOptions: { |
| 93 | + typeRoots: [path.join(repoRoot, 'node_modules/@types')], |
| 94 | + // Doc snippets are illustrative fragments, not production code: an |
| 95 | + // unused destructured result is normal and not a real defect here. |
| 96 | + noUnusedLocals: false, |
| 97 | + noUnusedParameters: false, |
| 98 | + paths: resolveExamplesSdkPaths(), |
| 99 | + }, |
| 100 | + // src/global.d.ts declares build-time ambient globals (e.g. __OWNER_FILES__) |
| 101 | + // that src/ files reference; examples/sdk/tsconfig.json includes it for the |
| 102 | + // same reason. |
| 103 | + include: ['*.ts', path.join(repoRoot, 'src/global.d.ts')], |
| 104 | + }), |
| 105 | + ); |
| 106 | + snippets.forEach((code, index) => { |
| 107 | + // `export {}` forces module scope so each snippet's declarations (and any |
| 108 | + // stubs injected below) can't collide with another snippet's globals. |
| 109 | + fs.writeFileSync(path.join(tmpDir, `snippet-${index}.ts`), `export {};\n${code}`); |
| 110 | + }); |
| 111 | + return tmpDir; |
| 112 | +} |
| 113 | + |
| 114 | +function runTsc(tmpDir: string): string { |
| 115 | + try { |
| 116 | + execFileSync( |
| 117 | + TSC_BIN, |
| 118 | + ['--noEmit', '-p', path.join(tmpDir, 'tsconfig.json'), '--pretty', 'false'], |
| 119 | + { |
| 120 | + encoding: 'utf8', |
| 121 | + cwd: tmpDir, |
| 122 | + }, |
| 123 | + ); |
| 124 | + return ''; |
| 125 | + } catch (error) { |
| 126 | + const { stdout, stderr } = error as { stdout?: string; stderr?: string }; |
| 127 | + return `${stdout ?? ''}${stderr ?? ''}`; |
| 128 | + } |
| 129 | +} |
| 130 | + |
| 131 | +// One auto-stub pass: for every "Cannot find name 'X'" diagnostic, declare `X` |
| 132 | +// (typed precisely if it's a known SDK-derived continuation, `any` otherwise) |
| 133 | +// at the top of that snippet file and recompile. A missing/renamed *import* |
| 134 | +// binding is a different diagnostic code and is never suppressed this way. |
| 135 | +function stubFreeNamesAndRecompile(tmpDir: string, firstPassOutput: string): string { |
| 136 | + const freeNamesByFile = new Map<string, Set<string>>(); |
| 137 | + for (const line of firstPassOutput.split('\n')) { |
| 138 | + const match = /^(snippet-\d+\.ts)\(\d+,\d+\): error TS2304: Cannot find name '([^']+)'/.exec( |
| 139 | + line, |
| 140 | + ); |
| 141 | + if (match?.[1] && match[2]) { |
| 142 | + const names = freeNamesByFile.get(match[1]) ?? new Set<string>(); |
| 143 | + names.add(match[2]); |
| 144 | + freeNamesByFile.set(match[1], names); |
| 145 | + } |
| 146 | + } |
| 147 | + if (freeNamesByFile.size === 0) return firstPassOutput; |
| 148 | + |
| 149 | + for (const [fileName, names] of freeNamesByFile) { |
| 150 | + const filePath = path.join(tmpDir, fileName); |
| 151 | + const stubs = [...names] |
| 152 | + .map((name) => `declare const ${name}: ${KNOWN_CONTINUATION_STUB_TYPES[name] ?? 'any'};`) |
| 153 | + .join('\n'); |
| 154 | + fs.writeFileSync( |
| 155 | + filePath, |
| 156 | + fs.readFileSync(filePath, 'utf8').replace('export {};\n', `export {};\n${stubs}\n`), |
| 157 | + ); |
| 158 | + } |
| 159 | + return runTsc(tmpDir); |
| 160 | +} |
| 161 | + |
| 162 | +function compileDocSnippets(snippets: string[]): string[] { |
| 163 | + const tmpDir = writeSnippetProgram(snippets); |
| 164 | + try { |
| 165 | + const output = stubFreeNamesAndRecompile(tmpDir, runTsc(tmpDir)); |
| 166 | + const diagnosticLines = output.split('\n').filter((line) => /error TS\d+:/.test(line)); |
| 167 | + return diagnosticLines.filter((line) => !KNOWN_DOC_GAPS.some((gap) => line.includes(gap))); |
| 168 | + } finally { |
| 169 | + fs.rmSync(tmpDir, { recursive: true, force: true }); |
| 170 | + } |
| 171 | +} |
| 172 | + |
| 173 | +test("every fenced ```ts snippet in client-api.md compiles against agent-device's real exports", () => { |
| 174 | + const snippets = extractTsSnippets(fs.readFileSync(CLIENT_API_DOC_PATH, 'utf8')); |
| 175 | + assert.ok( |
| 176 | + snippets.length > 0, |
| 177 | + `${CLIENT_API_DOC_PATH} has no fenced \`\`\`ts snippets to check.`, |
| 178 | + ); |
| 179 | + |
| 180 | + const failures = compileDocSnippets(snippets); |
| 181 | + assert.deepEqual( |
| 182 | + failures, |
| 183 | + [], |
| 184 | + `A \`\`\`ts snippet in ${CLIENT_API_DOC_PATH} no longer compiles against agent-device's real ` + |
| 185 | + `exports:\n${failures.join('\n')}\n` + |
| 186 | + 'Fix the snippet to match the current API, or update the SDK if the doc was right and the ' + |
| 187 | + 'API regressed. If this is a new, deliberately out-of-scope gap, add it to KNOWN_DOC_GAPS ' + |
| 188 | + 'with a comment explaining why.', |
| 189 | + ); |
| 190 | +}); |
0 commit comments