Skip to content

Commit 7d558de

Browse files
harden pinned browser CI setup and native fixture provenance (#494)
Signed-off-by: tulsi <[email protected]>
1 parent 47b0165 commit 7d558de

7 files changed

Lines changed: 412 additions & 207 deletions

File tree

‎.github/actions/setup/action.yml‎

Lines changed: 24 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,12 @@
11
name: Setup pinned tools and dependencies
2-
description: Hermit and frozen pnpm install.
2+
description: Hermit and frozen pnpm install; optionally verify the pinned browser image.
3+
inputs:
4+
browsers:
5+
description: Verify preinstalled browsers and libraries in the pinned CI image
6+
default: "false"
7+
browser-engine:
8+
description: Browser engine to verify (all, chromium or webkit)
9+
default: "all"
310
runs:
411
using: composite
512
steps:
@@ -11,10 +18,18 @@ runs:
1118
# A faster JS-only job must not freeze an incomplete cache for Rust jobs.
1219
key: hermit-${{ runner.os }}-${{ runner.arch }}-${{ github.job }}-${{ hashFiles('bin/**') }}
1320
- uses: cashapp/activate-hermit@cea9af7913204a965fd488637a8d1811bba2e616 # v1
21+
- name: Verify pinned tool entry points
22+
shell: bash
23+
run: |
24+
./bin/node --version
25+
./bin/pnpm --version
1426
- name: Resolve pnpm store
1527
id: pnpm
1628
shell: bash
17-
run: echo "path=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT"
29+
run: |
30+
store=$(./bin/pnpm store path --silent)
31+
test -n "$store"
32+
echo "path=$store" >> "$GITHUB_OUTPUT"
1833
- name: Cache pnpm store
1934
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
2035
with:
@@ -23,4 +38,10 @@ runs:
2338
restore-keys: pnpm-${{ runner.os }}-${{ runner.arch }}-
2439
- name: Install locked dependencies
2540
shell: bash
26-
run: pnpm install --frozen-lockfile
41+
run: ./bin/pnpm install --frozen-lockfile
42+
- name: Verify pinned browser image
43+
if: inputs.browsers == 'true'
44+
shell: bash
45+
env:
46+
PLAYWRIGHT_ENGINE: ${{ inputs.browser-engine }}
47+
run: ./bin/node scripts/check-browser-image.mjs

‎.github/workflows/ci.yml‎

Lines changed: 61 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -30,15 +30,15 @@ jobs:
3030
persist-credentials: false
3131
- uses: ./.github/actions/setup
3232
- name: Lint and types
33-
run: pnpm check
33+
run: ./bin/pnpm check
3434
# check already ran TypeScript; do not repeat pnpm build's typecheck.
3535
- name: Frontend build
36-
run: pnpm exec vite build
36+
run: ./bin/pnpm exec vite build
3737
- name: All Vitest tests
3838
env:
3939
# Bound contention between suites that sign and verify large fixtures.
4040
BUZZ_TEST_WORKERS: "2"
41-
run: node scripts/ci-test-report.mjs kind=vitest report=test-results/ci/vitest.json evidence=test-results/ci/vitest-timing.json title="Vitest tests" -- pnpm exec vitest run --reporter=default --reporter=json --outputFile=test-results/ci/vitest.json
41+
run: ./bin/node scripts/ci-test-report.mjs kind=vitest report=test-results/ci/vitest.json evidence=test-results/ci/vitest-timing.json title="Vitest tests" -- ./bin/pnpm exec vitest run --reporter=default --reporter=json --outputFile=test-results/ci/vitest.json
4242
- name: Vitest timing evidence
4343
if: always()
4444
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
@@ -66,14 +66,14 @@ jobs:
6666
key: native
6767
save-if: ${{ github.event_name == 'push' }}
6868
- name: Rust formatting
69-
run: cargo fmt --all --check
69+
run: ./bin/cargo fmt --all --check
7070
- name: Rust lint
71-
run: cargo clippy --workspace --locked --all-targets -- -D warnings
71+
run: ./bin/cargo clippy --workspace --locked --all-targets -- -D warnings
7272
# --workspace includes Tauri, unlike the default Cargo member selection.
7373
- name: All Rust tests and doctests
74-
run: cargo test --workspace --locked
74+
run: ./bin/cargo test --workspace --locked
7575
- name: All Node integration tests
76-
run: node --test tests/integration/*.test.mjs
76+
run: ./bin/node --test tests/integration/*.test.mjs
7777

7878
windows-native:
7979
if: github.event_name == 'workflow_dispatch'
@@ -107,29 +107,32 @@ jobs:
107107
measurements:
108108
if: github.event_name != 'workflow_dispatch'
109109
name: Browser measurements
110-
# https://playwright.dev/docs/ci#via-containers
110+
runs-on: ubuntu-24.04
111+
# Match package.json's Playwright version and pin immutable browser/OS layers.
111112
container:
112113
image: mcr.microsoft.com/playwright:v1.63.0-noble@sha256:eff16c30e6f3f4af0a03fa4b706120d5e9b0891c344a27d64559aff5900a4a27
113114
options: --init --ipc=host
114115
defaults:
115116
run:
116117
shell: bash
117-
runs-on: ubuntu-24.04
118+
env:
119+
HOME: /root
118120
timeout-minutes: 15
119121
steps:
120122
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
121123
with:
122124
persist-credentials: false
123-
- uses: ./.github/actions/setup
124-
# Container steps need their own Git trust for the host-owned checkout.
125-
- name: Trust checked-out workspace
125+
- name: Trust this container workspace
126126
run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
127+
- uses: ./.github/actions/setup
128+
with:
129+
browsers: "true"
127130
# Preserve Chromium -> WebKit measurement dependencies; never shard or
128131
# overlap the two measurement projects on this runner.
129132
- name: Serial Chromium and WebKit measurements
130133
env:
131134
PLAYWRIGHT_JSON_OUTPUT_FILE: test-results/browser/ci-report.json
132-
run: node scripts/ci-test-report.mjs kind=playwright report=test-results/browser/ci-report.json evidence=test-results/browser/ci-timing.json title="Browser measurements" -- pnpm test:browser:ci --project '*-measurements' --workers=1 --reporter=list,json
135+
run: ./bin/node scripts/ci-test-report.mjs kind=playwright report=test-results/browser/ci-report.json evidence=test-results/browser/ci-timing.json title="Browser measurements" -- ./bin/pnpm test:browser:ci --project '*-measurements' --workers=1 --reporter=list,json
133136
# Cases tagged @classic-scrollbars need Chromium's platform scrollbars,
134137
# which Linux always draws as classic ones. One file cannot join the
135138
# six-way engine matrix (empty shards are errors), so it runs here, and
@@ -138,7 +141,7 @@ jobs:
138141
- name: Chromium classic-scrollbar layout
139142
env:
140143
PLAYWRIGHT_JSON_OUTPUT_FILE: test-results/browser-classic-scrollbars/ci-report.json
141-
run: node scripts/ci-test-report.mjs kind=playwright report=test-results/browser-classic-scrollbars/ci-report.json evidence=test-results/browser-classic-scrollbars/ci-timing.json title="Browser classic scrollbars" -- pnpm test:browser:ci --project chromium-classic-scrollbars --no-deps --reporter=list,json
144+
run: ./bin/node scripts/ci-test-report.mjs kind=playwright report=test-results/browser-classic-scrollbars/ci-report.json evidence=test-results/browser-classic-scrollbars/ci-timing.json title="Browser classic scrollbars" -- ./bin/pnpm test:browser:ci --project chromium-classic-scrollbars --no-deps --reporter=list,json
142145
- name: Measurement evidence
143146
if: always()
144147
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
@@ -154,59 +157,53 @@ jobs:
154157
name: Native browser fixture
155158
runs-on: ubuntu-24.04
156159
timeout-minutes: 15
160+
outputs:
161+
artifact-id: ${{ steps.fixture-upload.outputs.artifact-id }}
157162
steps:
158163
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
159164
with:
160165
persist-credentials: false
161-
# Browser fixtures need only the pinned compiler, Cargo and standard library,
162-
# not Hermit's full Rust archive (including docs).
163-
- name: Install minimal pinned Rust for browser fixtures
164-
shell: bash
165-
run: |
166-
pins=(bin/.rust-*.pkg)
167-
test "${#pins[@]}" -eq 1
168-
test -L "${pins[0]}"
169-
version=${pins[0]#bin/.rust-}
170-
version=${version%.pkg}
171-
rustup toolchain install "$version" --profile minimal --no-self-update
172-
rustc_path=$(rustup which --toolchain "$version" rustc)
173-
toolchain_bin=$(dirname "$rustc_path")
174-
test -x "$toolchain_bin/rustc"
175-
test -x "$toolchain_bin/cargo"
176-
echo "RUSTUP_TOOLCHAIN=$version" >> "$GITHUB_ENV"
177-
echo "HERMIT_PREPEND_PATH=$toolchain_bin" >> "$GITHUB_ENV"
178-
# This journey uses a real native plugin-manager bridge. Provision it
179-
# before Playwright starts: a cold Rust install/build is test setup, not
180-
# browser behavior, and must not consume the journey's subprocess budget.
166+
- uses: ./.github/actions/setup
181167
- uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2
182168
with:
183169
key: browser-fixture
184170
save-if: ${{ github.event_name == 'push' }}
185171
- name: Build native browser fixture
186-
run: cargo build --locked -p buzzodz-plugins --example fixture-bridge
172+
timeout-minutes: 8
173+
run: ./bin/cargo build --locked -p buzzodz-plugins --example fixture-bridge --target-dir target/browser-fixture
174+
- name: Package same-revision native fixture
175+
run: |
176+
mkdir -p target/browser-artifact
177+
cp target/browser-fixture/debug/examples/fixture-bridge target/browser-artifact/
178+
git rev-parse HEAD > target/browser-artifact/revision
179+
cd target/browser-artifact
180+
sha256sum fixture-bridge > SHA256SUMS
181+
tar -cf ../browser-fixture.tar fixture-bridge revision SHA256SUMS
187182
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
183+
id: fixture-upload
188184
with:
189-
name: native-browser-fixture
190-
path: target/debug/examples/fixture-bridge
185+
name: browser-fixture-${{ github.sha }}-${{ github.run_attempt }}
186+
path: target/browser-fixture.tar
191187
if-no-files-found: error
192188
retention-days: 1
193189

194190
browser:
191+
if: github.event_name != 'workflow_dispatch'
192+
name: Browser journeys (${{ matrix.engine }}, ${{ matrix.shard }}/6)
193+
# Measurements stay isolated on their own runner, not on our critical path.
194+
# CI required still requires their success independently of these jobs.
195195
needs: [browser_fixture]
196-
# https://playwright.dev/docs/ci#via-containers
196+
runs-on: ubuntu-24.04
197+
# Match package.json's Playwright version and pin immutable browser/OS layers.
197198
container:
198199
image: mcr.microsoft.com/playwright:v1.63.0-noble@sha256:eff16c30e6f3f4af0a03fa4b706120d5e9b0891c344a27d64559aff5900a4a27
199200
options: --init --ipc=host
200201
defaults:
201202
run:
202203
shell: bash
203204
env:
204-
BUZZ_BROWSER_FIXTURE: ${{ github.workspace }}/.browser-fixture/fixture-bridge
205-
if: github.event_name != 'workflow_dispatch'
206-
name: Browser journeys (${{ matrix.engine }}, ${{ matrix.shard }}/6)
207-
# Measurements stay isolated on their own runner, not on our critical path.
208-
# CI required still requires their success independently of these jobs.
209-
runs-on: ubuntu-24.04
205+
HOME: /root
206+
BUZZ_BROWSER_FIXTURE_DIR: target/ci-browser-fixture
210207
# Six file-level shards split the large file groups that remain together
211208
# with four or five shards, leaving room for setup inside the same deadline.
212209
timeout-minutes: 15
@@ -219,22 +216,30 @@ jobs:
219216
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
220217
with:
221218
persist-credentials: false
222-
- uses: ./.github/actions/setup
223-
# Container steps need their own Git trust for the host-owned checkout.
224-
- name: Trust checked-out workspace
219+
- name: Trust this container workspace
225220
run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
226-
- uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
221+
- uses: ./.github/actions/setup
222+
with:
223+
browsers: "true"
224+
browser-engine: ${{ matrix.engine }}
225+
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
227226
with:
228-
name: native-browser-fixture
229-
path: .browser-fixture
230-
- name: Restore fixture executable permission
231-
run: chmod +x .browser-fixture/fixture-bridge
227+
artifact-ids: ${{ needs.browser_fixture.outputs.artifact-id }}
228+
merge-multiple: true
229+
path: target/ci-browser-fixture
230+
- name: Verify native browser fixture
231+
run: |
232+
cd target/ci-browser-fixture
233+
tar -xf browser-fixture.tar
234+
test "$(cat revision)" = "$(git rev-parse HEAD)"
235+
sha256sum --check SHA256SUMS
236+
test -x fixture-bridge
232237
# CI required enforces measurement success. --no-deps avoids repeating
233238
# measurements here; the matrix covers both engines on separate runners.
234239
- name: Functional journeys
235240
env:
236241
PLAYWRIGHT_JSON_OUTPUT_FILE: test-results/browser/ci-report.json
237-
run: node scripts/ci-test-report.mjs kind=playwright report=test-results/browser/ci-report.json evidence=test-results/browser/ci-timing.json title="Browser journeys (${{ matrix.engine }}, ${{ matrix.shard }}/6)" -- pnpm test:browser:ci --project ${{ matrix.engine }} --no-deps --shard=${{ matrix.shard }}/6 --reporter=list,json
242+
run: ./bin/node scripts/ci-test-report.mjs kind=playwright report=test-results/browser/ci-report.json evidence=test-results/browser/ci-timing.json title="Browser journeys (${{ matrix.engine }}, ${{ matrix.shard }}/6)" -- ./bin/pnpm test:browser:ci --project ${{ matrix.engine }} --no-deps --shard=${{ matrix.shard }}/6 --reporter=list,json
238243
- name: Browser evidence
239244
if: always()
240245
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
@@ -246,7 +251,7 @@ jobs:
246251
required:
247252
name: ${{ github.event_name == 'workflow_dispatch' && 'Automatic CI (not requested)' || 'CI required' }}
248253
if: always() && github.event_name != 'workflow_dispatch'
249-
needs: [javascript, native, measurements, browser]
254+
needs: [javascript, native, measurements, browser_fixture, browser]
250255
runs-on: ubuntu-24.04
251256
timeout-minutes: 2
252257
steps:
@@ -255,8 +260,9 @@ jobs:
255260
JAVASCRIPT: ${{ needs.javascript.result }}
256261
NATIVE: ${{ needs.native.result }}
257262
MEASUREMENTS: ${{ needs.measurements.result }}
263+
FIXTURE: ${{ needs.browser_fixture.result }}
258264
BROWSER: ${{ needs.browser.result }}
259265
run: |
260-
for result in "$JAVASCRIPT" "$NATIVE" "$MEASUREMENTS" "$BROWSER"; do
266+
for result in "$JAVASCRIPT" "$NATIVE" "$MEASUREMENTS" "$FIXTURE" "$BROWSER"; do
261267
test "$result" = success || exit 1
262268
done

‎docs/contributing.md‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -357,16 +357,25 @@ the complete suite still runs with `pnpm test` / `just scan`:
357357
- **Browser measurements:** Chromium then WebKit, serially on an isolated runner.
358358
- **Browser journeys:** twelve runners (Chromium and WebKit, six file-level shards
359359
per engine), each with two workers. They start alongside measurements on separate
360-
runners; `CI required` still requires both lanes. A separate Ubuntu job builds
361-
the native plugin-manager fixture using the repository Rust pin and uploads it
362-
for all twelve shards. Shards wait for that job, restore executable permission,
363-
and pass its path through `BUZZ_BROWSER_FIXTURE`; local journeys still build
364-
with Cargo. No measurements are repeated on shards and no retries hide failures.
360+
runners; `CI required` still requires both lanes. A separate required Ubuntu
361+
job builds the native plugin-manager fixture once with Hermit's pinned Cargo.
362+
It uploads a tar with executable permission, checkout revision and SHA-256
363+
checksum; shards download by exact same-run artifact ID and verify all three
364+
before running. A missing artifact fails CI rather than rebuilding. Local
365+
non-CI journeys retain the locked Cargo build. Each browser test uses its own
366+
mutable fixture home. No measurements are repeated on shards and no retries
367+
hide failures.
365368
- Both browser lanes use the version-matched, digest-pinned
366369
[Playwright Docker image](https://playwright.dev/docs/docker), which supplies
367370
browsers and Linux libraries without per-job apt provisioning. Follow the
368371
[CI container guidance](https://playwright.dev/docs/ci#via-containers).
369372
Update both image references and digests when upgrading `@playwright/test`.
373+
Setup verifies installed Playwright against image metadata and launches the
374+
selected engine (both for measurements) before tests; it never downloads a
375+
missing browser. Hermit pins Node/pnpm through explicit `./bin/` entry points
376+
and fails closed if the pnpm store path cannot be resolved. Containers use
377+
`HOME=/root` and trust only their exact checked-out workspace. Native host
378+
jobs keep their normal toolchain and library setup.
370379
- **CI required:** fails unless every automatic Linux lane and every browser shard succeeds,
371380
including cancellation or an unexpectedly skipped lane. Configure this status
372381
as a required repository check; the workflow does not change branch protection.

‎scripts/check-browser-image.mjs‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
import assert from "node:assert/strict";
2+
import { readFileSync } from "node:fs";
3+
import { createRequire } from "node:module";
4+
import { fileURLToPath } from "node:url";
5+
6+
const require = createRequire(import.meta.url);
7+
8+
export function imageEngines(image, installedVersion, engine) {
9+
assert.equal(
10+
image.driverVersion,
11+
installedVersion,
12+
"CI browser image must match installed Playwright",
13+
);
14+
assert.equal(
15+
image.dockerImageName,
16+
`mcr.microsoft.com/playwright:v${installedVersion}-noble`,
17+
);
18+
assert.ok(
19+
["all", "chromium", "webkit"].includes(engine),
20+
"Unsupported browser engine",
21+
);
22+
return engine === "all" ? ["chromium", "webkit"] : [engine];
23+
}
24+
25+
async function main() {
26+
const engines = imageEngines(
27+
JSON.parse(readFileSync("/ms-playwright/.docker-info", "utf8")),
28+
require("@playwright/test/package.json").version,
29+
process.env.PLAYWRIGHT_ENGINE ?? "all",
30+
);
31+
const playwright = require("@playwright/test");
32+
for (const engine of engines) {
33+
// Launch, do not just check a path: missing OS libraries must fail setup.
34+
const browser = await playwright[engine].launch();
35+
try {
36+
const page = await browser.newPage();
37+
await page.setContent("<p>Browser ready</p>");
38+
assert.equal(await page.locator("p").textContent(), "Browser ready");
39+
console.log(`${engine}: pinned image browser launched`);
40+
} finally {
41+
await browser.close();
42+
}
43+
}
44+
}
45+
if (process.argv[1] === fileURLToPath(import.meta.url)) await main();

‎tests/browser/conversation.spec.mjs‎

Lines changed: 2 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import { createServer } from "vite";
33
import react from "@vitejs/plugin-react";
44
import { fileURLToPath } from "node:url";
55
import { run } from "./run-command.mjs";
6+
import { nativeFixture } from "./native-fixture.mjs";
67
import { mkdtemp, cp, readFile, writeFile, rm } from "node:fs/promises";
78
import { join } from "node:path";
89
import { tmpdir } from "node:os";
@@ -51,22 +52,7 @@ test("independent packed author consumer and native-installed contribution survi
5152
run("pnpm", ["build"], source);
5253
const code = await readFile(join(source, "dist/plugin.js"), "utf8");
5354
expect(code).not.toMatch(/^import\s|^export.*from\s/m);
54-
// CI supplies the Ubuntu-built bridge; local runs retain the Cargo build.
55-
let binary = process.env.BUZZ_BROWSER_FIXTURE;
56-
if (!binary) {
57-
run("cargo", [
58-
"build",
59-
"--locked",
60-
"-p",
61-
"buzzodz-plugins",
62-
"--example",
63-
"fixture-bridge",
64-
]);
65-
const metadata = JSON.parse(
66-
run("cargo", ["metadata", "--no-deps", "--format-version=1"]),
67-
);
68-
binary = join(metadata.target_directory, "debug/examples/fixture-bridge");
69-
}
55+
const binary = nativeFixture();
7056
const home = join(temp, "home");
7157
const native = (op, ...args) =>
7258
JSON.parse(run(binary, [home, op, ...args]));

0 commit comments

Comments
 (0)