Skip to content

Commit f2ded46

Browse files
test(web): add evidence scripts for default model effort fix
Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 1ce968c commit f2ded46

9 files changed

Lines changed: 625 additions & 0 deletions

File tree

‎.evidence/pr-22193/diff.sh‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
#!/usr/bin/env bash
2+
# For every *-before.png in <dir> with a matching *-after.png, writes two diff images:
3+
# *-diff.png pixel diff: changed pixels in red over a faded copy of the before image
4+
# *-sbs.png labeled side-by-side: Before | After | Diff
5+
# Usage: diff.sh <stillsDir> [outDir]
6+
set -u
7+
dir="$1"; out="${2:-$1/diffs}"
8+
mkdir -p "$out"
9+
for before in "$dir"/*-before.png; do
10+
after="${before%-before.png}-after.png"
11+
[ -f "$after" ] || continue
12+
stem="$(basename "${before%-before.png}")"
13+
# compare exits 1 when images differ; that is the expected case.
14+
pixels=$(compare -metric AE -fuzz 2% -highlight-color red -lowlight-color '#ffffff40' "$before" "$after" "$out/$stem-diff.png" 2>&1 >/dev/null || true)
15+
montage -label 'BEFORE (3b95e78)' "$before" -label 'AFTER (1ce968c)' "$after" -label "DIFF ($pixels px differ)" "$out/$stem-diff.png" \
16+
-tile 3x1 -geometry +12+12 -pointsize 22 -background '#e8e8e8' "$out/$stem-sbs.png"
17+
echo "$stem $pixels"
18+
done

‎.evidence/pr-22193/lib.mjs‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
// Shared browser helpers for the PR #22193 evidence drivers.
2+
import { createRequire } from "node:module";
3+
import { execFileSync } from "node:child_process";
4+
import { existsSync, readFileSync } from "node:fs";
5+
import { dirname, join } from "node:path";
6+
import { fileURLToPath } from "node:url";
7+
8+
const here = dirname(fileURLToPath(import.meta.url));
9+
// Playwright comes from apps/web's dev dependencies in this checkout.
10+
const require = createRequire(join(here, "../../apps/web/package.json"));
11+
export const { chromium } = require("playwright");
12+
13+
// Web origin from the newest [dev-runner] line of a dev server log.
14+
export function webOriginFromLog(logFile) {
15+
const ports = [...readFileSync(logFile, "utf8").matchAll(/webPort=(\d+)/g)];
16+
if (ports.length === 0) throw new Error(`no [dev-runner] line in ${logFile}`);
17+
return `http://localhost:${ports.at(-1)[1]}`;
18+
}
19+
20+
// Mints a short-lived pairing URL for the dev server that owns <worktree>/.t3.
21+
// The token is never logged; the resulting session is cached in authFile.
22+
export function mintPairingUrl(worktree) {
23+
const out = execFileSync(
24+
"node",
25+
[
26+
join(worktree, "apps/server/src/bin.ts"),
27+
"pair",
28+
"--base-dir",
29+
join(worktree, ".t3"),
30+
"--ttl",
31+
"10m",
32+
],
33+
{ cwd: worktree, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] },
34+
);
35+
const match = out.match(/Pairing URL: (\S+)/);
36+
if (!match) throw new Error("pair command printed no URL");
37+
return match[1];
38+
}
39+
40+
export async function openApp({
41+
worktree,
42+
authFile,
43+
webOrigin,
44+
width = 1440,
45+
height = 900,
46+
theme = "light",
47+
videoDir,
48+
}) {
49+
const browser = await chromium.launch();
50+
const contextOptions = {
51+
viewport: { width, height },
52+
colorScheme: theme,
53+
deviceScaleFactor: 1,
54+
...(videoDir ? { recordVideo: { dir: videoDir, size: { width, height } } } : {}),
55+
};
56+
let pairing = null;
57+
if (!existsSync(authFile)) {
58+
pairing = mintPairingUrl(worktree);
59+
const context = await browser.newContext({ viewport: { width, height } });
60+
const page = await context.newPage();
61+
await page.goto(pairing);
62+
await page.waitForURL((url) => !url.pathname.startsWith("/pair"), { timeout: 30_000 });
63+
await context.storageState({ path: authFile });
64+
await context.close();
65+
}
66+
const context = await browser.newContext({ ...contextOptions, storageState: authFile });
67+
const page = await context.newPage();
68+
const origin = pairing ? new URL(pairing).origin : webOrigin;
69+
return { browser, context, page, origin };
70+
}
71+
72+
// On-screen caption so recordings narrate themselves.
73+
export async function caption(page, text) {
74+
await page.evaluate((value) => {
75+
let el = document.getElementById("__ev_caption");
76+
if (!el) {
77+
el = document.createElement("div");
78+
el.id = "__ev_caption";
79+
el.style.cssText =
80+
"position:fixed;left:50%;bottom:16px;transform:translateX(-50%);z-index:2147483647;" +
81+
"background:rgba(0,0,0,.82);color:#fff;font:600 15px/1.35 system-ui,sans-serif;" +
82+
"padding:8px 14px;border-radius:8px;max-width:92vw;pointer-events:none;white-space:pre-wrap";
83+
document.body.appendChild(el);
84+
}
85+
el.textContent = value;
86+
}, text);
87+
}
88+
89+
export async function gotoSettingsGeneral(page, origin) {
90+
await page.goto(`${origin}/settings/general`);
91+
await page.locator("#default-model").first().waitFor({ timeout: 30_000 });
92+
// Provider snapshots arrive after the first paint; give them up to 30s.
93+
await page
94+
.waitForFunction(
95+
() =>
96+
!document.querySelector("#default-model")?.textContent?.includes("No providers available"),
97+
null,
98+
{ timeout: 30_000 },
99+
)
100+
.catch(() => {});
101+
await page.waitForTimeout(2000);
102+
}
103+
104+
export function modelRow(page) {
105+
return page.locator("#default-model").first();
106+
}

‎.evidence/pr-22193/new-thread.mjs‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
import { setTimeout as sleep } from "node:timers/promises";
2+
3+
// Opens a fresh draft on the new-thread screen, picking the first project when asked.
4+
export async function openNewThread(page, origin) {
5+
await page.goto(origin);
6+
await page
7+
.getByRole("button", { name: "New thread", exact: true })
8+
.first()
9+
.waitFor({ timeout: 90_000 });
10+
await sleep(1500);
11+
await page.getByRole("button", { name: "New thread", exact: true }).first().click();
12+
// New thread asks for a project; take the first one.
13+
const picker = page.locator('[data-slot="command-dialog-backdrop"]');
14+
await picker.waitFor({ timeout: 5000 }).catch(() => {});
15+
if (await picker.count()) {
16+
await sleep(800);
17+
await page.keyboard.press("Enter");
18+
await picker.waitFor({ state: "detached", timeout: 10_000 }).catch(() => {});
19+
}
20+
await page
21+
.locator('[data-composer-shortcut="composer.effort"], [data-chat-provider-model-picker="true"]')
22+
.first()
23+
.waitFor({ timeout: 30_000 });
24+
await sleep(2500);
25+
}

‎.evidence/pr-22193/run-all.sh‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
#!/usr/bin/env bash
2+
# Records rows on both builds in parallel: run-all.sh <before-worktree> <after-worktree> [outDir] [rows...]
3+
set -u
4+
here="$(cd "$(dirname "$0")" && pwd)"
5+
before="$1"; after="$2"; out="${3:-/tmp/ev-22193}"; shift 3 || shift $#
6+
rows=("${@:-R1a R1d R1e R2a R3a R1f R1b R1c R2b}")
7+
for row in ${rows[@]}; do
8+
node "$here/run-row.mjs" "$row" before "$before" "$out" > "$out/$row-before.out" 2>&1 &
9+
b=$!
10+
node "$here/run-row.mjs" "$row" after "$after" "$out" > "$out/$row-after.out" 2>&1 &
11+
a=$!
12+
wait $b; echo "$row before exit=$?"
13+
wait $a; echo "$row after exit=$?"
14+
done

0 commit comments

Comments
 (0)