Skip to content

Commit d218cec

Browse files
ci(studio): gates read the base the run measured, and drag frames on a quiet runner (#4872)
* ci(studio): measure drag smoothness on a quiet runner, with a blank-page control * ci(studio): gates read the base the run measured, not the base branch tip * style(ci): format the base-file test
1 parent c3b6e55 commit d218cec

5 files changed

Lines changed: 164 additions & 23 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 5 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -851,14 +851,7 @@ jobs:
851851
- name: Read the base branch's work-count ceilings
852852
# A ceiling may not disappear, or rise without a new reason, against the branch this PR merges into.
853853
if: github.event_name == 'pull_request'
854-
env:
855-
BASE_REF: ${{ github.base_ref }}
856-
run: |
857-
git fetch --no-tags --depth=1 origin "$BASE_REF"
858-
if ! git show FETCH_HEAD:packages/studio/tests/e2e/perf-ceilings.json > /tmp/base-perf-ceilings.json; then
859-
rm -f /tmp/base-perf-ceilings.json
860-
echo "The base branch has no perf-ceilings.json yet"
861-
fi
854+
run: bash scripts/ci/base-file.sh packages/studio/tests/e2e/perf-ceilings.json /tmp/base-perf-ceilings.json
862855
- name: Run both arms of the timeline viewport gate
863856
run: |
864857
set -euo pipefail
@@ -1020,14 +1013,9 @@ jobs:
10201013
# Studio and the producer both run in the engine's pinned headless shell.
10211014
- run: node packages/cli/dist/cli.js browser ensure
10221015
- name: Read the base branch's edit accuracy baseline
1023-
env:
1024-
BASE_REF: ${{ github.base_ref }}
10251016
run: |
1026-
git fetch --no-tags --depth=1 origin "$BASE_REF"
1027-
if ! git show FETCH_HEAD:packages/studio/tests/e2e/edit-accuracy/baseline.json > /tmp/base-edit-accuracy.json; then
1028-
echo '{"cases":{}}' > /tmp/base-edit-accuracy.json
1029-
echo "The base branch has no edit accuracy baseline yet"
1030-
fi
1017+
bash scripts/ci/base-file.sh packages/studio/tests/e2e/edit-accuracy/baseline.json /tmp/base-edit-accuracy.json
1018+
[ -f /tmp/base-edit-accuracy.json ] || echo '{"cases":{}}' > /tmp/base-edit-accuracy.json
10311019
- name: Run this shard, then re-run twice each case whose verdict differs from the base branch
10321020
run: |
10331021
set -euo pipefail
@@ -1062,14 +1050,9 @@ jobs:
10621050
with:
10631051
node-version: 22
10641052
- name: Read the base branch's edit accuracy baseline
1065-
env:
1066-
BASE_REF: ${{ github.base_ref }}
10671053
run: |
1068-
git fetch --no-tags --depth=1 origin "$BASE_REF"
1069-
if ! git show FETCH_HEAD:packages/studio/tests/e2e/edit-accuracy/baseline.json > /tmp/base-edit-accuracy.json; then
1070-
echo '{"cases":{}}' > /tmp/base-edit-accuracy.json
1071-
echo "The base branch has no edit accuracy baseline yet"
1072-
fi
1054+
bash scripts/ci/base-file.sh packages/studio/tests/e2e/edit-accuracy/baseline.json /tmp/base-edit-accuracy.json
1055+
[ -f /tmp/base-edit-accuracy.json ] || echo '{"cases":{}}' > /tmp/base-edit-accuracy.json
10731056
- uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
10741057
with:
10751058
pattern: edit-accuracy-shard-*
Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
name: Studio drag frames
2+
3+
permissions:
4+
contents: read
5+
6+
env:
7+
HYPERFRAMES_NO_TELEMETRY: "1"
8+
9+
# Measures drag smoothness on a runner with nothing else on it. The bench measures a blank page per case
10+
# in the same run, and the box only counts as quiet if that control dropped no frame anywhere.
11+
on:
12+
workflow_dispatch:
13+
inputs:
14+
ref:
15+
description: Commit, branch or tag to measure
16+
default: main
17+
pull_request:
18+
paths:
19+
- ".github/workflows/studio-drag-frames.yml"
20+
21+
jobs:
22+
drag-frames:
23+
name: "Studio: drag frames"
24+
runs-on: ubuntu-latest
25+
timeout-minutes: 60
26+
env:
27+
BENCH_REF: ${{ inputs.ref || 'main' }}
28+
steps:
29+
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
30+
with:
31+
ref: ${{ env.BENCH_REF }}
32+
lfs: true
33+
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
34+
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
35+
with:
36+
node-version: 22
37+
- run: corepack enable
38+
- run: corepack prepare [email protected] --activate
39+
- uses: ./.github/actions/prepare-ffmpeg-bin
40+
- run: bash scripts/ci/install-workspace-dependencies.sh
41+
- run: bun run build
42+
- run: node packages/cli/dist/cli.js browser ensure
43+
- name: Run the drag cases five times
44+
working-directory: packages/studio
45+
run: |
46+
set -euo pipefail
47+
echo "Measuring $(git rev-parse HEAD)"
48+
for n in 1 2 3 4 5; do
49+
bun tests/e2e/edit-accuracy/run.mjs --grid pr \
50+
--filter '^(move|resize|rotate|crop)-(none|tween)-px-r0-root-z100$' \
51+
--jobs 1 --port 5961 --cli "$GITHUB_WORKSPACE/packages/cli/dist/cli.js" \
52+
--out "$RUNNER_TEMP/drag-frames/run-$n"
53+
done
54+
- name: Require a quiet box (the blank page dropped no frame)
55+
run: |
56+
# shellcheck disable=SC2016 # the backticks are JavaScript template strings
57+
node -e '
58+
const fs = require("fs"), dir = process.env.RUNNER_TEMP + "/drag-frames";
59+
let noisy = 0;
60+
for (const run of fs.readdirSync(dir)) {
61+
const { cases } = JSON.parse(fs.readFileSync(`${dir}/${run}/results.json`, "utf8"));
62+
for (const c of cases) {
63+
const dropped = c.smooth?.control?.dropped;
64+
if (dropped !== 0) { noisy++; console.log(`${run} ${c.id} control dropped=${dropped}`); }
65+
}
66+
}
67+
if (noisy) { console.log(`::error::the blank page dropped frames, or went unmeasured, in ${noisy} case(s); this box is not quiet`); process.exit(1); }
68+
console.log("the blank page dropped 0 frames in every case of every run");
69+
'
70+
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
71+
if: always()
72+
with:
73+
name: studio-drag-frames
74+
path: ${{ runner.temp }}/drag-frames
75+
if-no-files-found: error
76+
retention-days: 30

‎package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@
5252
"player:perf": "bun run --filter @hyperframes/player perf",
5353
"format:check": "oxfmt --check .",
5454
"knip": "knip",
55-
"test:scripts": "node --import tsx --test scripts/animejs-v4-guidance.test.mjs scripts/check-tracked-artifacts.test.mjs scripts/check-registry-set-delta.test.mjs scripts/check-no-main-deletions.test.mjs scripts/check-pr-captures.test.mjs scripts/check-comment-citations.test.mjs scripts/comments-workflow.test.mjs scripts/comment-ratchet.test.mjs scripts/check-docs-snippet-motion.test.mjs scripts/registry-target-paths.test.mjs scripts/check-workspace-contracts.test.mjs scripts/check-media-use-copy-parity.test.mjs scripts/check-svg-sanitize-parity.test.mjs scripts/check-media-use-svg-sanitize-generated.test.mjs scripts/check-package-cycles.test.mjs scripts/check-cli-process-ownership.test.mjs scripts/check-large-files.test.mjs scripts/prune-fallow-audit-caches.test.mjs scripts/package-subpaths.test.mjs scripts/validate-release-channel.test.mjs scripts/publish-workflow.test.mjs scripts/pr-edit-concurrency.test.mjs scripts/merge-queue-workflows.test.mjs scripts/install-workspace-dependencies.test.mjs scripts/draft-changelog.test.ts scripts/set-version.test.ts scripts/release-prepare.test.ts scripts/cli-options.test.ts scripts/changelog-weekly.test.ts scripts/claude-plugin-compression.test.ts scripts/catalog-payload-assets.test.ts scripts/host-registry-assets.test.ts scripts/catalog-preview-temp.test.ts scripts/catalog-hosted-files.test.ts scripts/player-cdn-pin.test.ts scripts/studio-runtime-smoke.test.mjs scripts/verify-packed-manifests.test.mjs scripts/lint-skills.test.mjs scripts/creator-editing-recipes.test.mjs packages/gcp-cloud-run/check-dockerfile-workspaces.test.mjs packages/core/scripts/writeGeneratedFile.test.ts scripts/catalog-publication.test.mjs scripts/ci/resolve-workflow-pr.test.mjs scripts/check-catalog-source-pr.test.mjs scripts/generate-registry-items.test.ts scripts/catalog-drift.test.ts scripts/catalog-fetch-mirror.test.ts scripts/catalog-script-inlining.test.ts scripts/catalog-detail.test.ts scripts/generate-catalog-pages.test.ts scripts/verify-catalog-payloads.test.ts scripts/registry-skill-files.test.ts scripts/creator-editing-capabilities.test.mjs scripts/generate-catalog-previews.test.ts scripts/registry-primitive-payloads.test.ts registry/components/pan-stations/pan-stations.test.mjs && vitest run scripts/catalog/ scripts/contrastRatchet.test.ts scripts/generate-catalog-payloads.test.ts",
55+
"test:scripts": "node --import tsx --test scripts/animejs-v4-guidance.test.mjs scripts/check-tracked-artifacts.test.mjs scripts/check-registry-set-delta.test.mjs scripts/check-no-main-deletions.test.mjs scripts/check-pr-captures.test.mjs scripts/check-comment-citations.test.mjs scripts/comments-workflow.test.mjs scripts/comment-ratchet.test.mjs scripts/check-docs-snippet-motion.test.mjs scripts/registry-target-paths.test.mjs scripts/check-workspace-contracts.test.mjs scripts/check-media-use-copy-parity.test.mjs scripts/check-svg-sanitize-parity.test.mjs scripts/check-media-use-svg-sanitize-generated.test.mjs scripts/check-package-cycles.test.mjs scripts/check-cli-process-ownership.test.mjs scripts/check-large-files.test.mjs scripts/prune-fallow-audit-caches.test.mjs scripts/package-subpaths.test.mjs scripts/validate-release-channel.test.mjs scripts/publish-workflow.test.mjs scripts/pr-edit-concurrency.test.mjs scripts/merge-queue-workflows.test.mjs scripts/install-workspace-dependencies.test.mjs scripts/draft-changelog.test.ts scripts/set-version.test.ts scripts/release-prepare.test.ts scripts/cli-options.test.ts scripts/changelog-weekly.test.ts scripts/claude-plugin-compression.test.ts scripts/catalog-payload-assets.test.ts scripts/host-registry-assets.test.ts scripts/catalog-preview-temp.test.ts scripts/catalog-hosted-files.test.ts scripts/player-cdn-pin.test.ts scripts/studio-runtime-smoke.test.mjs scripts/verify-packed-manifests.test.mjs scripts/lint-skills.test.mjs scripts/creator-editing-recipes.test.mjs packages/gcp-cloud-run/check-dockerfile-workspaces.test.mjs packages/core/scripts/writeGeneratedFile.test.ts scripts/catalog-publication.test.mjs scripts/ci/resolve-workflow-pr.test.mjs scripts/ci/base-file.test.mjs scripts/check-catalog-source-pr.test.mjs scripts/generate-registry-items.test.ts scripts/catalog-drift.test.ts scripts/catalog-fetch-mirror.test.ts scripts/catalog-script-inlining.test.ts scripts/catalog-detail.test.ts scripts/generate-catalog-pages.test.ts scripts/verify-catalog-payloads.test.ts scripts/registry-skill-files.test.ts scripts/creator-editing-capabilities.test.mjs scripts/generate-catalog-previews.test.ts scripts/registry-primitive-payloads.test.ts registry/components/pan-stations/pan-stations.test.mjs && vitest run scripts/catalog/ scripts/contrastRatchet.test.ts scripts/generate-catalog-payloads.test.ts",
5656
"typecheck:scripts": "tsc --noEmit -p scripts/tsconfig.json",
5757
"test:skills": "HYPERFRAMES_MEDIA_HOME_REQUIRED=1 node --import ./packages/cli/scripts/test-home.mjs --test 'skills/**/*.test.mjs' 'packages/cli/src/media-use/**/*.test.mjs'",
5858
"generate:previews": "tsx scripts/generate-template-previews.ts",

‎scripts/ci/base-file.sh‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
#!/usr/bin/env bash
2+
# base-file.sh <path> <out>: writes <path> as it was on the base this pull_request run measured, the first parent of
3+
# the checked-out merge commit. The base branch tip moves while a run is in flight, so it is never read. A base without
4+
# the file removes <out>; a failed fetch fails the step.
5+
set -euo pipefail
6+
base="$(git cat-file -p HEAD | awk '/^parent /{print $2; exit}')"
7+
echo "Reading $1 from the base this run measured: $base"
8+
git fetch --no-tags --depth=1 origin "$base"
9+
if git cat-file -e "$base:$1" 2>/dev/null; then
10+
git show "$base:$1" > "$2"
11+
else
12+
rm -f "$2"
13+
echo "The base has no $1 yet"
14+
fi

‎scripts/ci/base-file.test.mjs‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
import assert from "node:assert/strict";
2+
import { execFileSync } from "node:child_process";
3+
import { existsSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs";
4+
import { tmpdir } from "node:os";
5+
import { join } from "node:path";
6+
import { test } from "node:test";
7+
8+
const script = join(import.meta.dirname, "base-file.sh");
9+
const git = (cwd, ...args) =>
10+
execFileSync(
11+
"git",
12+
["-c", "user.email=t@t", "-c", "user.name=t", "-c", "commit.gpgsign=false", ...args],
13+
{
14+
cwd,
15+
encoding: "utf8",
16+
},
17+
).trim();
18+
19+
// A PR run checks out the merge commit shallowly; the base branch then moves before the gate reads its baseline.
20+
function raceFixture(baseline) {
21+
const root = mkdtempSync(join(tmpdir(), "base-file-"));
22+
const origin = join(root, "origin.git");
23+
const work = join(root, "work");
24+
git(root, "init", "-q", "--bare", origin);
25+
git(root, "init", "-q", "-b", "main", work);
26+
git(work, "remote", "add", "origin", origin);
27+
if (baseline) writeFileSync(join(work, "baseline.json"), baseline);
28+
writeFileSync(join(work, "readme"), "base");
29+
git(work, "add", ".");
30+
git(work, "commit", "-qm", "base");
31+
const base = git(work, "rev-parse", "HEAD");
32+
git(work, "switch", "-qc", "pr");
33+
writeFileSync(join(work, "pr"), "change");
34+
git(work, "add", ".");
35+
git(work, "commit", "-qm", "pr");
36+
git(work, "switch", "-q", "main");
37+
git(work, "merge", "-q", "--no-ff", "-m", "merge", "pr");
38+
const merge = git(work, "rev-parse", "HEAD");
39+
git(work, "push", "-q", "origin", `${merge}:refs/pull/1/merge`);
40+
git(work, "reset", "-q", "--hard", base);
41+
writeFileSync(join(work, "baseline.json"), '{"total":852}');
42+
git(work, "add", ".");
43+
git(work, "commit", "-qm", "base moves on");
44+
git(work, "push", "-q", "origin", "HEAD:main");
45+
const checkout = join(root, "checkout");
46+
git(root, "init", "-q", checkout);
47+
git(checkout, "remote", "add", "origin", `file://${origin}`);
48+
git(checkout, "fetch", "-q", "--depth=1", "origin", "refs/pull/1/merge");
49+
git(checkout, "checkout", "-q", "FETCH_HEAD");
50+
return { checkout, base, out: join(root, "out.json") };
51+
}
52+
53+
test("reads the base the run measured, not the base branch tip that moved since", () => {
54+
const { checkout, base, out } = raceFixture('{"total":846}');
55+
const log = execFileSync("bash", [script, "baseline.json", out], {
56+
cwd: checkout,
57+
encoding: "utf8",
58+
});
59+
assert.equal(readFileSync(out, "utf8"), '{"total":846}');
60+
assert.match(log, new RegExp(`measured: ${base}`));
61+
});
62+
63+
test("a base without the file leaves no output", () => {
64+
const { checkout, out } = raceFixture(null);
65+
writeFileSync(out, "stale");
66+
execFileSync("bash", [script, "baseline.json", out], { cwd: checkout, encoding: "utf8" });
67+
assert.equal(existsSync(out), false);
68+
});

0 commit comments

Comments
 (0)