Skip to content

Commit 070b964

Browse files
feat(cli): check replays an unchanged project's last pass and takes --no-craft (#7398)
* feat(cli): check returns an unchanged project's last pass in about a second * fix(cli): the check cache hashes files without a stat race and its tests run before the signal test * test(cli): the warm check test compares two real runs, not a run and its cached pass * feat(cli): check --no-craft skips the craft pass for a render preflight * fix(cli): the check cache keys on timeout, env and large files and never replays a moved tree * style(cli): format the check cache * fix(cli): the large-file cache test waits for the clock tick it depends on * fix(cli): the check cache keys on the browser and on scripts under .hyperframes * fix(cli): the check cache ignores the video copies the check itself touches
1 parent 12906e0 commit 070b964

7 files changed

Lines changed: 405 additions & 6 deletions

File tree

‎docs/packages/cli.mdx‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1271,7 +1271,9 @@ cover every finding from the full check.
12711271
| `--tolerance` | Allowed overflow in px before reporting (default 2) |
12721272
| `--timeout` | Render-ready budget in ms; also raises page navigation above its 10s floor (default 45000, as snapshot and render) |
12731273
| `--budget <ms\|90s\|8m>` | Stop the whole check this long after it starts (at least 30s) and report what it found, with `check_budget_reached` naming the phase it was in. For hosts with their own time limit. |
1274+
| `--no-cache` | Run the check even when the project files, flags, CLI and browser are unchanged since the last pass. Without it, an unchanged project returns that pass in about a second, marked as cached; remote scripts and assets are not fetched again for that. A failure is never cached, and a project with `node_modules` or a linked directory is never replayed. |
12741275
| `--no-contrast` | Skip the WCAG pass while iterating |
1276+
| `--no-craft` | Skip the craft pass (dead stretches, unread lines, silent film), the longest part of a full check. The report marks it skipped. Use it for a render preflight; keep it on for the full check before a release. |
12751277
| `--strict` | Exit non-zero on warnings too (default: errors only) |
12761278
| `--caption-zone "<x0=..;y0=..;x1=..;y1=..>"` | Opt-in band gate. Flags a text element's DOM box that overlaps the fractional band. Optional `severity` and `seek`. |
12771279
| `--frame-check` | Opt-in out-of-frame detection for `img`, `svg`, `video`, and `canvas` |

‎packages/cli/src/commands/check.test.ts‎

Lines changed: 208 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,15 @@
11
import type { CheckProgress } from "../utils/checkProgress.js";
22
import { runCommand } from "citty";
3-
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
3+
import {
4+
mkdirSync,
5+
mkdtempSync,
6+
readFileSync,
7+
rmSync,
8+
statSync,
9+
symlinkSync,
10+
utimesSync,
11+
writeFileSync,
12+
} from "node:fs";
413
import { tmpdir } from "node:os";
514
import { join } from "node:path";
615
import { afterEach, describe, expect, it, vi } from "vitest";
@@ -544,6 +553,24 @@ it("--budget sets when the whole check stops, its report time kept back, counted
544553
expect(runPipeline.mock.calls[1]?.[1].budget).toBeUndefined();
545554
});
546555

556+
it("--no-craft reaches the pipeline as craft: false, and the craft pass is on without it", async () => {
557+
const { report } = await runScenario(fakeDriver());
558+
const runPipeline = vi.fn(async (_project: ProjectDir, _options: CheckOptions) => report);
559+
vi.spyOn(console, "log").mockImplementation(() => undefined);
560+
const command = createCheckCommand({
561+
resolveProject: () => PROJECT,
562+
runPipeline,
563+
prepareMedia: vi.fn(async () => []),
564+
withMeta: (value) => value,
565+
});
566+
567+
await runCommand(command, { rawArgs: ["--json", "--no-craft"] });
568+
await runCommand(command, { rawArgs: ["--json"] });
569+
570+
expect(runPipeline.mock.calls[0]?.[1].craft).toBe(false);
571+
expect(runPipeline.mock.calls[1]?.[1].craft).toBe(true);
572+
});
573+
547574
it("parses the caption-zone grammar and enables the frame gate", async () => {
548575
const { report } = await runScenario(fakeDriver());
549576
const runPipeline = vi.fn(async (_project: ProjectDir, _options: CheckOptions) => report);
@@ -3046,6 +3073,15 @@ describe("check pipeline", () => {
30463073
expect(checkExitCode(strict.report)).toBe(0);
30473074
});
30483075

3076+
it("--no-craft never samples the craft frames and the report says the pass was skipped", async () => {
3077+
const driver = craftDriver();
3078+
const { report } = await runScenario(driver, { craft: false });
3079+
3080+
expect(driver.collectCraftFrame).not.toHaveBeenCalled();
3081+
expect(report.craft.enabled).toBe(false);
3082+
expect(report.craft.findings).toEqual([]);
3083+
});
3084+
30493085
it("seeks each craft frame with WebGL draws on, so a redrawn canvas reads as moving", async () => {
30503086
const driver = craftDriver();
30513087
await runScenario(driver);
@@ -3991,6 +4027,177 @@ it("applies transition sample limits within the requested range", async () => {
39914027
expect(report.layout.transitionSamplesDropped).toBe(3);
39924028
});
39934029

4030+
describe("check cache", () => {
4031+
const dirs: string[] = [];
4032+
afterEach(() => {
4033+
vi.unstubAllEnvs();
4034+
for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true });
4035+
});
4036+
4037+
function setup(report: CheckReport) {
4038+
const browser: { identity: string | undefined } = { identity: "chrome:1" };
4039+
const dir = mkdtempSync(join(tmpdir(), "hf-check-cache-"));
4040+
dirs.push(dir);
4041+
writeFileSync(join(dir, "index.html"), "<html>one</html>");
4042+
const runPipeline = vi.fn(async (_project: ProjectDir, _options: CheckOptions) => report);
4043+
const command = createCheckCommand({
4044+
resolveProject: () => ({ dir, name: "p", indexPath: join(dir, "index.html") }),
4045+
runPipeline,
4046+
prepareMedia: vi.fn(async () => []),
4047+
withMeta: (value) => value,
4048+
browserIdentity: async () => browser.identity,
4049+
});
4050+
vi.spyOn(console, "log").mockImplementation(() => undefined);
4051+
return {
4052+
dir,
4053+
browser,
4054+
runPipeline,
4055+
run: (...flags: string[]) => runCommand(command, { rawArgs: ["--json", ...flags] }),
4056+
};
4057+
}
4058+
4059+
it("answers an unchanged tree from the last pass without running the pipeline", async () => {
4060+
const { runPipeline, run } = setup(reportWithFindings());
4061+
await run();
4062+
await run();
4063+
expect(runPipeline).toHaveBeenCalledTimes(1);
4064+
});
4065+
4066+
it("re-runs after one byte of the composition changes", async () => {
4067+
const { dir, runPipeline, run } = setup(reportWithFindings());
4068+
await run();
4069+
writeFileSync(join(dir, "index.html"), "<html>onf</html>");
4070+
await run();
4071+
expect(runPipeline).toHaveBeenCalledTimes(2);
4072+
});
4073+
4074+
it("never stores a failure", async () => {
4075+
const { runPipeline, run } = setup(reportWithFindings({ ok: false }));
4076+
await run();
4077+
await run();
4078+
expect(runPipeline).toHaveBeenCalledTimes(2);
4079+
});
4080+
4081+
it("never stores a pass whose browser did not run", async () => {
4082+
const { runPipeline, run } = setup(reportWithFindings({ browserSkipped: true }));
4083+
await run();
4084+
await run();
4085+
expect(runPipeline).toHaveBeenCalledTimes(2);
4086+
});
4087+
4088+
it("does not store a pass for a tree that changed while it was being checked", async () => {
4089+
const { dir, runPipeline, run } = setup(reportWithFindings());
4090+
runPipeline.mockImplementationOnce(async () => {
4091+
writeFileSync(join(dir, "index.html"), "<html>written by someone else</html>");
4092+
return reportWithFindings();
4093+
});
4094+
await run();
4095+
writeFileSync(join(dir, "index.html"), "<html>one</html>");
4096+
await run();
4097+
expect(runPipeline).toHaveBeenCalledTimes(2);
4098+
});
4099+
4100+
it("re-runs a large file changed to the same size with its mtime put back", async () => {
4101+
const { dir, runPipeline, run } = setup(reportWithFindings());
4102+
const big = join(dir, "clip.bin");
4103+
const mtime = 1_700_000_000;
4104+
writeFileSync(big, Buffer.alloc(3 * 1024 * 1024, 1));
4105+
utimesSync(big, mtime, mtime);
4106+
await run();
4107+
const writtenAt = statSync(big).ctimeMs;
4108+
await vi.waitFor(
4109+
() => {
4110+
writeFileSync(big, Buffer.alloc(3 * 1024 * 1024, 2));
4111+
utimesSync(big, mtime, mtime);
4112+
if (statSync(big).ctimeMs === writtenAt) throw new Error("the clock has not ticked");
4113+
},
4114+
{ interval: 1 },
4115+
);
4116+
await run();
4117+
expect(runPipeline).toHaveBeenCalledTimes(2);
4118+
});
4119+
4120+
it("never caches a project with node_modules or a linked directory", async () => {
4121+
const { dir, runPipeline, run } = setup(reportWithFindings());
4122+
mkdirSync(join(dir, "node_modules"));
4123+
await run();
4124+
await run();
4125+
rmSync(join(dir, "node_modules"), { recursive: true });
4126+
mkdirSync(join(dir, "real"));
4127+
symlinkSync(join(dir, "real"), join(dir, "linked"), "dir");
4128+
await run();
4129+
await run();
4130+
expect(runPipeline).toHaveBeenCalledTimes(4);
4131+
});
4132+
4133+
it("re-runs when --timeout or the runtime URL changes", async () => {
4134+
const { runPipeline, run } = setup(reportWithFindings());
4135+
vi.stubEnv("HYPERFRAME_RUNTIME_URL", "http://localhost:1/runtime.js");
4136+
await run();
4137+
await run("--timeout", "20s");
4138+
vi.stubEnv("HYPERFRAME_RUNTIME_URL", "http://localhost:2/runtime.js");
4139+
await run();
4140+
vi.unstubAllEnvs();
4141+
expect(runPipeline).toHaveBeenCalledTimes(3);
4142+
});
4143+
4144+
it("re-runs when the browser binary changes, and never caches without one", async () => {
4145+
const { browser, runPipeline, run } = setup(reportWithFindings());
4146+
await run();
4147+
browser.identity = "chrome:2";
4148+
await run();
4149+
browser.identity = undefined;
4150+
await run();
4151+
await run();
4152+
expect(runPipeline).toHaveBeenCalledTimes(4);
4153+
});
4154+
4155+
it("re-runs when a script under .hyperframes changes, but not when the CLI's own files do", async () => {
4156+
const { dir, runPipeline, run } = setup(reportWithFindings());
4157+
mkdirSync(join(dir, ".hyperframes", "artifacts"), { recursive: true });
4158+
writeFileSync(join(dir, ".hyperframes", "probe.js"), "console.log(1)");
4159+
await run();
4160+
writeFileSync(join(dir, ".hyperframes", "artifacts", "report.json"), "{}");
4161+
await run();
4162+
expect(runPipeline).toHaveBeenCalledTimes(1);
4163+
writeFileSync(join(dir, ".hyperframes", "probe.js"), "console.error(1)");
4164+
await run();
4165+
expect(runPipeline).toHaveBeenCalledTimes(2);
4166+
});
4167+
4168+
it("still hits when the check touches its video copies under .transcode-cache", async () => {
4169+
const { dir, runPipeline, run } = setup(reportWithFindings());
4170+
const copy = join(dir, ".transcode-cache", "clip.mp4");
4171+
mkdirSync(join(dir, ".transcode-cache"));
4172+
writeFileSync(copy, Buffer.alloc(3 * 1024 * 1024, 1));
4173+
runPipeline.mockImplementationOnce(async () => {
4174+
utimesSync(copy, new Date(), new Date());
4175+
return reportWithFindings();
4176+
});
4177+
await run();
4178+
await run();
4179+
expect(runPipeline).toHaveBeenCalledTimes(1);
4180+
});
4181+
4182+
it("never caches a project whose .hyperframes links to another directory", async () => {
4183+
const { dir, runPipeline, run } = setup(reportWithFindings());
4184+
mkdirSync(join(dir, "real"));
4185+
mkdirSync(join(dir, ".hyperframes"));
4186+
symlinkSync(join(dir, "real"), join(dir, ".hyperframes", "linked"), "dir");
4187+
await run();
4188+
await run();
4189+
expect(runPipeline).toHaveBeenCalledTimes(2);
4190+
});
4191+
4192+
it("re-runs when the flags differ, or with --no-cache, or under --budget", async () => {
4193+
const { runPipeline, run } = setup(reportWithFindings());
4194+
await run();
4195+
await run("--strict");
4196+
await run("--no-cache");
4197+
await run("--budget", "8m");
4198+
expect(runPipeline).toHaveBeenCalledTimes(4);
4199+
});
4200+
});
39944201
describe("check stopped by a signal", () => {
39954202
it("SIGTERM prints the report so far, then exits 143", async () => {
39964203
const { deps, runBrowserCheck } = dependencies(fakeDriver());

‎packages/cli/src/commands/check.ts‎

Lines changed: 49 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,14 @@ import { parseTimeoutMs } from "../utils/timeoutArg.js";
4444
import { failureMessage, jsonFailure } from "../utils/jsonError.js";
4545
import { recordLastCheck } from "../telemetry/config.js";
4646
import { failureCodes } from "../utils/checkFailureCodes.js";
47+
import {
48+
browserIdentity,
49+
checkCacheKey,
50+
checkCacheable,
51+
readCachedPass,
52+
storePass,
53+
} from "../utils/checkCache.js";
54+
import { VERSION } from "../version.js";
4755

4856
export const examples: Example[] = [
4957
["Run the full verification gate", "hyperframes check"],
@@ -61,6 +69,7 @@ export interface CheckCommandDependencies {
6169
runPipeline(project: ProjectDir, options: CheckOptions): Promise<SavedCheckReport>;
6270
prepareMedia(project: ProjectDir, options: CheckOptions): Promise<PreparedMediaFile[]>;
6371
withMeta(value: object): object;
72+
browserIdentity?(): Promise<string | undefined>;
6473
}
6574

6675
const DEFAULT_COMMAND_DEPENDENCIES: CheckCommandDependencies = {
@@ -227,6 +236,12 @@ const CHECK_COMMAND_ARGS = {
227236
description:
228237
"Stop the whole check this long after it starts (ms, or 90s, 8m; at least 30s) and report what it found so far",
229238
},
239+
cache: {
240+
type: "boolean",
241+
description:
242+
"Return the stored result when the project, options and CLI are unchanged since the last pass (pass --no-cache to always re-run)",
243+
default: true,
244+
},
230245
timeout: {
231246
type: "string",
232247
description:
@@ -238,6 +253,12 @@ const CHECK_COMMAND_ARGS = {
238253
description: "Run the WCAG AA contrast pass (enabled by default)",
239254
default: true,
240255
},
256+
craft: {
257+
type: "boolean",
258+
description:
259+
"Run the craft pass (dead stretches, unread lines, silent films; enabled by default). --no-craft skips it, e.g. for a render preflight",
260+
default: true,
261+
},
241262
strict: {
242263
type: "boolean",
243264
description: "Exit non-zero on warnings too",
@@ -313,9 +334,7 @@ export function createCheckCommand(
313334
return;
314335
}
315336
recordLastCheck({ at: new Date().toISOString(), codes: {} });
316-
const report = await withProjectLock(project.dir, "check", () =>
317-
dependencies.runPipeline(project, options),
318-
);
337+
const report = await checkOrReplay(dependencies, project, options);
319338
recordLastCheck({
320339
at: new Date().toISOString(),
321340
exitCode: checkExitCode(report),
@@ -349,6 +368,27 @@ export function createCheckCommand(
349368
});
350369
}
351370

371+
async function checkOrReplay(
372+
dependencies: CheckCommandDependencies,
373+
project: ProjectDir,
374+
options: CheckOptions,
375+
): Promise<SavedCheckReport> {
376+
const identify = dependencies.browserIdentity ?? browserIdentity;
377+
return withProjectLock(project.dir, "check", async () => {
378+
const keyNow = async () =>
379+
checkCacheable(options)
380+
? checkCacheKey(project.dir, options, VERSION, await identify())
381+
: undefined;
382+
const key = await keyNow();
383+
const cached = key ? readCachedPass(project.dir, key) : undefined;
384+
if (cached) return cached;
385+
const report = await dependencies.runPipeline(project, options);
386+
const treeUnchangedByTheRun = key !== undefined && (await keyNow()) === key;
387+
if (treeUnchangedByTheRun) storePass(project.dir, key, report);
388+
return report;
389+
});
390+
}
391+
352392
function reportCheckFailure(
353393
error: unknown,
354394
asJson: boolean,
@@ -450,7 +490,9 @@ function parseCheckOptions(args: Record<string, unknown>): CheckOptions {
450490
tolerance: nonNegativeNumber(args.tolerance, DEFAULT_CHECK_OPTIONS.tolerance),
451491
timeout: parseTimeoutMs(args.timeout, DEFAULT_CHECK_OPTIONS.timeout, 500),
452492
budget: parseBudget(args.budget),
493+
cache: args.cache !== false,
453494
contrast: args.contrast !== false,
495+
craft: args.craft !== false,
454496
...(checkReview()?.skipContrastInMotion ? { skipContrastInMotion: true } : {}),
455497
strict: args.strict === true,
456498
snapshots: args.snapshots === true,
@@ -693,6 +735,10 @@ function browserSkippedLine(report: CheckReport): string {
693735
}
694736

695737
function printHumanReport(report: CheckReport): void {
738+
if (report.cached)
739+
console.log(
740+
`${c.dim("◇")} Nothing changed since the pass at ${report.cached.at}; showing that result. Remote scripts and assets were not fetched again. Run with --no-cache to check again.`,
741+
);
696742
if (report.scope) {
697743
console.log(
698744
`Scope: ${report.scope.range ? report.scope.range.join("–") + "s" : "full timeline"}; ${report.scope.hiddenFindings} findings hidden by component filter. Counts/status include hidden findings.`,

0 commit comments

Comments
 (0)