Skip to content

Commit b7d72c9

Browse files
feat(cli): check --budget stops the whole check in time and reports what it found (#7245)
* feat(cli): check --budget stops the whole check in time and reports what it found * fix(cli): keep check's comment share where it was * fix(cli): a --budget that runs out before any check step is waiting is not an unhandled rejection
1 parent c1977fa commit b7d72c9

9 files changed

Lines changed: 183 additions & 10 deletions

File tree

‎docs/packages/cli.mdx‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1249,6 +1249,7 @@ cover every finding from the full check.
12491249
| `--collapse-static` / `--no-collapse-static` | Fold a repeated static finding into one row (default: on) |
12501250
| `--tolerance` | Allowed overflow in px before reporting (default 2) |
12511251
| `--timeout` | Render-ready budget in ms; also raises page navigation above its 10s floor (default 45000, as snapshot and render) |
1252+
| `--budget <ms\|90s\|8m>` | Stop the whole check this long after it starts (at least 30s) and report what it found, with `check_budget_reached` naming the phase it was in. For hosts with their own time limit. |
12521253
| `--no-contrast` | Skip the WCAG pass while iterating |
12531254
| `--strict` | Exit non-zero on warnings too (default: errors only) |
12541255
| `--caption-zone "<x0=..;y0=..;x1=..;y1=..>"` | Opt-in band gate. Flags a text element's DOM box that overlaps the fractional band. Optional `severity` and `seek`. |

‎packages/cli/src/commands/check.test.ts‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -518,6 +518,28 @@ it("waits for the composition as long as snapshot and render do unless --timeout
518518
expect(runPipeline.mock.calls[1]?.[1].timeout).toBe(20_000);
519519
});
520520

521+
it("--budget sets when the whole check stops, its report time kept back, counted from the command's start", async () => {
522+
const { report } = await runScenario(fakeDriver());
523+
const runPipeline = vi.fn(async (_project: ProjectDir, _options: CheckOptions) => report);
524+
vi.spyOn(console, "log").mockImplementation(() => undefined);
525+
const command = createCheckCommand({
526+
resolveProject: () => PROJECT,
527+
runPipeline,
528+
prepareMedia: vi.fn(async () => []),
529+
withMeta: (value) => value,
530+
});
531+
532+
const startedAt = Date.now();
533+
await runCommand(command, { rawArgs: ["--json", "--budget", "8m"] });
534+
await runCommand(command, { rawArgs: ["--json"] });
535+
536+
const budget = runPipeline.mock.calls[0]?.[1].budget;
537+
expect(budget?.ms).toBe(480_000);
538+
expect(budget?.stopAt).toBeGreaterThanOrEqual(startedAt + 470_000);
539+
expect(budget?.stopAt).toBeLessThanOrEqual(Date.now() + 470_000);
540+
expect(runPipeline.mock.calls[1]?.[1].budget).toBeUndefined();
541+
});
542+
521543
it("parses the caption-zone grammar and enables the frame gate", async () => {
522544
const { report } = await runScenario(fakeDriver());
523545
const runPipeline = vi.fn(async (_project: ProjectDir, _options: CheckOptions) => report);
@@ -2497,6 +2519,58 @@ describe("check pipeline", () => {
24972519
}
24982520
});
24992521

2522+
it("--budget ends a check that is still within its idle window and keeps what its finished samples found", async () => {
2523+
vi.useFakeTimers();
2524+
try {
2525+
let aborted!: AbortSignal;
2526+
const driver = fakeDriver({
2527+
collectLayout: vi.fn(async (time: number) => [
2528+
layoutIssue("warning", { time, code: "container_overflow" }),
2529+
]),
2530+
collectOverlapGrid: vi.fn(
2531+
() =>
2532+
new Promise<never>((_, reject) =>
2533+
aborted.addEventListener("abort", () => reject(aborted.reason)),
2534+
),
2535+
),
2536+
});
2537+
const { deps } = dependencies(driver);
2538+
deps.runBrowserCheck = async (_project, options, motion, deadline) => {
2539+
aborted = deadline!.signal;
2540+
deadline!.start();
2541+
return runAuditGrid(
2542+
driver,
2543+
options,
2544+
motion,
2545+
undefined,
2546+
deadline!.enter,
2547+
undefined,
2548+
deadline!.keepPartial,
2549+
);
2550+
};
2551+
const budget = { ms: 70_000, stopAt: Date.now() + 60_000 };
2552+
const pending = runCheckPipeline(
2553+
PROJECT,
2554+
{ ...DEFAULT_CHECK_OPTIONS, timeout: 400_000, budget },
2555+
deps,
2556+
);
2557+
await vi.advanceTimersByTimeAsync(60_000);
2558+
const report = await pending;
2559+
const reached = report.runtime.findings.find(
2560+
(finding) => finding.code === "check_budget_reached",
2561+
);
2562+
expect(reached?.message).toMatch(
2563+
/^Check reached its --budget of 70 s while running the overlap pass .*Seconds per phase: /,
2564+
);
2565+
expect(report.ok).toBe(false);
2566+
expect(report.layout.findings.some((finding) => finding.code === "container_overflow")).toBe(
2567+
true,
2568+
);
2569+
} finally {
2570+
vi.useRealTimers();
2571+
}
2572+
});
2573+
25002574
it("--strict flips a warnings-only result from exit 0 to exit 1", async () => {
25012575
const warningDriver = () =>
25022576
fakeDriver({

‎packages/cli/src/commands/check.ts‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -215,6 +215,11 @@ const CHECK_COMMAND_ARGS = {
215215
description: "Allowed pixel overflow before reporting an issue (default: 2)",
216216
default: "2",
217217
},
218+
budget: {
219+
type: "string",
220+
description:
221+
"Stop the whole check this long after it starts (ms, or 90s, 8m; at least 30s) and report what it found so far",
222+
},
218223
timeout: {
219224
type: "string",
220225
description:
@@ -348,6 +353,13 @@ function reportCheckFailure(
348353
}
349354

350355
const REPORT_GRACE_MS = 5_000;
356+
const MIN_BUDGET_MS = 30_000;
357+
const BUDGET_REPORT_MS = 10_000;
358+
359+
function parseBudget(raw: unknown): CheckOptions["budget"] {
360+
const ms = parseTimeoutMs(raw, 0, MIN_BUDGET_MS, console.error, "--budget");
361+
return ms > 0 ? { ms, stopAt: Date.now() + ms - BUDGET_REPORT_MS } : undefined;
362+
}
351363

352364
function untilReported(): { done: () => void } {
353365
let done = (): void => undefined;
@@ -423,6 +435,7 @@ function parseCheckOptions(args: Record<string, unknown>): CheckOptions {
423435
collapseStatic: args["collapse-static"] !== false,
424436
tolerance: nonNegativeNumber(args.tolerance, DEFAULT_CHECK_OPTIONS.tolerance),
425437
timeout: parseTimeoutMs(args.timeout, DEFAULT_CHECK_OPTIONS.timeout, 500),
438+
budget: parseBudget(args.budget),
426439
contrast: args.contrast !== false,
427440
...(checkReview()?.skipContrastInMotion ? { skipContrastInMotion: true } : {}),
428441
strict: args.strict === true,

‎packages/cli/src/utils/checkBrowser.test.ts‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1637,6 +1637,43 @@ describe("preResolveHostileMediaProxies", () => {
16371637
}
16381638
});
16391639

1640+
it("gives the copies only what is left of the check's --budget when that is under the copy budget", async () => {
1641+
vi.useFakeTimers();
1642+
try {
1643+
const projectDir = mkProjectDir();
1644+
mocks.scanProjectMediaCodecMap.mockResolvedValue({
1645+
"/clip.mov": {
1646+
codecName: "hevc",
1647+
browserHostile: true,
1648+
representativeMime: null,
1649+
hasAlpha: false,
1650+
},
1651+
});
1652+
mocks.resolveProxy.mockImplementation(
1653+
() =>
1654+
new Promise<string>((_, reject) =>
1655+
mocks.proxySignal?.addEventListener("abort", () => reject(new Error("aborted"))),
1656+
),
1657+
);
1658+
vi.spyOn(console, "error").mockImplementation(() => undefined);
1659+
const deadline = new CheckDeadline(undefined, { ms: 40_000, stopAt: Date.now() + 30_000 });
1660+
1661+
const pending = runBrowserCheck(
1662+
{ ...PROJECT, dir: projectDir, indexPath: `${projectDir}/index.html` },
1663+
DEFAULT_CHECK_OPTIONS,
1664+
{ kind: "none" },
1665+
runAuditGrid,
1666+
deadline,
1667+
);
1668+
const outcome = expect(pending).rejects.toThrow(/took longer than 30 s: clip\.mov/);
1669+
await vi.advanceTimersByTimeAsync(30_000);
1670+
await outcome;
1671+
await deadline.close();
1672+
} finally {
1673+
vi.useRealTimers();
1674+
}
1675+
});
1676+
16401677
it("stops reporting a copy once it is done, so a finished one is not read as stuck", async () => {
16411678
vi.useFakeTimers();
16421679
try {

‎packages/cli/src/utils/checkBrowser.ts‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -443,7 +443,12 @@ export async function runBrowserCheck(
443443
findings: drafts,
444444
failedPaths,
445445
limits,
446-
} = await preResolveHostileMediaProxies(project.dir, html, options.autoProxy, COPY_BUDGET_MS);
446+
} = await preResolveHostileMediaProxies(
447+
project.dir,
448+
html,
449+
options.autoProxy,
450+
Math.min(COPY_BUDGET_MS, deadline?.remainingMs() ?? COPY_BUDGET_MS),
451+
);
447452
deadline?.enter("launch");
448453
const server = await serveStaticProjectHtml(
449454
project.dir,

‎packages/cli/src/utils/checkDeadline.ts‎

Lines changed: 35 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,14 @@ export class CheckCopiesUnfinishedError extends Error {
4949
}
5050
}
5151

52+
export class CheckBudgetError extends Error {
53+
constructor(budgetMs: number, where: string) {
54+
super(
55+
`Check reached its --budget of ${budgetMs / 1000} s${where || "."} The report holds what it found before then.`,
56+
);
57+
}
58+
}
59+
5260
export class CheckStoppedError extends Error {
5361
constructor(signal: string) {
5462
super(`Check stopped before finishing (${signal}).`);
@@ -71,10 +79,28 @@ export class CheckDeadline {
7179
private readonly spent = new Map<CheckPhase, number>();
7280
private readonly partials: Array<() => Partial<CheckBrowserResult>> = [];
7381

74-
constructor(private readonly idleMs = CHECK_IDLE_MS) {
82+
private readonly budgetTimer: NodeJS.Timeout | undefined;
83+
private readonly stopAt: number | undefined;
84+
85+
constructor(
86+
private readonly idleMs = CHECK_IDLE_MS,
87+
budget?: { ms: number; stopAt: number },
88+
) {
7589
this.expiry = new Promise((_, reject) => {
7690
this.rejectExpiry = reject;
7791
});
92+
this.expiry.catch(() => undefined);
93+
this.budgetTimer =
94+
budget &&
95+
setTimeout(
96+
() => this.expire(new CheckBudgetError(budget.ms, this.where())),
97+
Math.max(0, budget.stopAt - Date.now()),
98+
);
99+
this.stopAt = budget?.stopAt;
100+
}
101+
102+
remainingMs(): number | undefined {
103+
return this.stopAt === undefined ? undefined : Math.max(0, this.stopAt - Date.now());
78104
}
79105

80106
keepPartial = (read: () => Partial<CheckBrowserResult>): void => {
@@ -99,13 +125,18 @@ export class CheckDeadline {
99125
}
100126

101127
private describe(): string {
128+
const where = this.where();
129+
return where && ` (no progress for ${this.idleMs / 1000} s after Chrome was ready),${where}`;
130+
}
131+
132+
private where(): string {
102133
if (!this.work) return "";
103134
const open = Date.now() - this.work.since;
104135
const parts = [...this.spent].map(([phase, ms]) => {
105136
const seconds = ((phase === this.work?.phase ? ms + open : ms) / 1000).toFixed(1);
106137
return `${PHASE_LABELS[phase]} ${seconds}${BEFORE_DEADLINE.has(phase) ? " (before the deadline)" : ""}`;
107138
});
108-
return ` (no progress for ${this.idleMs / 1000} s after Chrome was ready), while ${PHASE_LABELS[this.work.phase]}${this.work.detail ? ` (${this.work.detail})` : ""}. Seconds per phase: ${parts.join(", ")}.`;
139+
return ` while ${PHASE_LABELS[this.work.phase]}${this.work.detail ? ` (${this.work.detail})` : ""}. Seconds per phase: ${parts.join(", ")}.`;
109140
}
110141

111142
start = (): void => {
@@ -125,6 +156,7 @@ export class CheckDeadline {
125156

126157
private expire(error: Error): void {
127158
clearTimeout(this.timer);
159+
clearTimeout(this.budgetTimer);
128160
this.expired = error;
129161
this.rejectExpiry(error);
130162
this.abort.abort(error);
@@ -173,6 +205,7 @@ export class CheckDeadline {
173205
async close(): Promise<void> {
174206
this.phase = "closed";
175207
clearTimeout(this.timer);
208+
clearTimeout(this.budgetTimer);
176209
const closes = await Promise.allSettled([...this.browsers].map(closeManagedBrowser));
177210
const errors = closes
178211
.filter((entry) => entry.status === "rejected")

‎packages/cli/src/utils/checkPipeline.ts‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import { DEFAULT_CONFIG } from "@hyperframes/engine";
22
import { BrowserCleanupError, onStopSignal } from "../browser/launch.js";
33
import type { CheckProgress } from "./checkProgress.js";
44
import {
5+
CheckBudgetError,
56
CheckDeadline,
67
CheckCopiesUnfinishedError,
78
CheckDeadlineError,
@@ -1734,7 +1735,7 @@ export async function runCheckPipeline(
17341735
]
17351736
: [];
17361737

1737-
const deadline = new CheckDeadline(idleWindow(options.timeout));
1738+
const deadline = new CheckDeadline(idleWindow(options.timeout), options.budget);
17381739
const forgetStop = onStopSignal(deadline.stop);
17391740
let outcome: { kind: "report"; report: CheckReport } | { kind: "failed"; error: unknown };
17401741
try {
@@ -1839,15 +1840,15 @@ async function withFindingCrops(
18391840
deadline: CheckDeadline,
18401841
): Promise<CheckReport> {
18411842
const cropRequests = selectFindingCropRequests(report);
1842-
if (cropRequests.length === 0) return report;
1843+
if (cropRequests.length === 0 || deadline.signal.aborted) return report;
18431844
try {
18441845
deadline.enter("crops");
18451846
const findingFiles = await deadline.run(() =>
18461847
dependencies.captureFindingCrops(project, options, cropRequests, deadline),
18471848
);
18481849
return { ...report, snapshots: { ...report.snapshots, findingFiles } };
18491850
} catch (error) {
1850-
if (error instanceof CheckDeadlineError) {
1851+
if (isCheckExpiry(error)) {
18511852
const finding = runtimeFailure(error);
18521853
return { ...report, ok: false, runtime: section([...report.runtime.findings, finding]) };
18531854
}
@@ -1881,7 +1882,7 @@ async function withOptionalFindingFrames(
18811882
const captured = await deadline.run(() => capture(project, options, requests, deadline));
18821883
return withFindingFrameFields(report, plan, captured);
18831884
} catch (error) {
1884-
if (error instanceof CheckDeadlineError || error instanceof CheckStoppedError) {
1885+
if (isCheckExpiry(error)) {
18851886
const finding = runtimeFailure(error);
18861887
return { ...report, ok: false, runtime: section([...report.runtime.findings, finding]) };
18871888
}
@@ -1891,6 +1892,11 @@ async function withOptionalFindingFrames(
18911892
}
18921893
}
18931894

1895+
const isCheckExpiry = (error: unknown): boolean =>
1896+
error instanceof CheckDeadlineError ||
1897+
error instanceof CheckStoppedError ||
1898+
error instanceof CheckBudgetError;
1899+
18941900
const MAX_FINDING_CROPS = 12;
18951901

18961902
/** Which error findings get a `finding-NN-<code>.png` crop for `check --snapshots`:
@@ -2263,7 +2269,9 @@ function runtimeFailure(error: unknown, code?: string): CheckFinding {
22632269
code ??
22642270
(error instanceof CheckCopiesUnfinishedError
22652271
? "media_copies_unfinished"
2266-
: "check_runtime_failure"),
2272+
: error instanceof CheckBudgetError
2273+
? "check_budget_reached"
2274+
: "check_runtime_failure"),
22672275
"error",
22682276
normalizeErrorMessage(error),
22692277
error instanceof CheckDeadlineError ? (error.sourceFile ?? "index.html") : "index.html",

‎packages/cli/src/utils/checkTypes.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ export interface CheckOptions {
2323
collapseStatic: boolean;
2424
tolerance: number;
2525
timeout: number;
26+
budget?: { ms: number; stopAt: number };
2627
contrast: boolean;
2728
skipContrastInMotion?: boolean;
2829
strict: boolean;

‎packages/cli/src/utils/timeoutArg.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,13 @@ export function parseTimeoutMs(
77
fallback: number,
88
floorMs = 0,
99
warn: (message: string) => void = console.error,
10+
flag = "--timeout",
1011
): number {
1112
if (raw === undefined || raw === null || raw === "") return fallback;
1213
const text = String(raw).trim();
1314
const match = /^(\d+(?:\.\d+)?)\s*(ms|s|m)?$/i.exec(text);
1415
if (!match) {
15-
warn(`[hyperframes] --timeout "${text}" is not a duration; using ${fallback} ms.`);
16+
warn(`[hyperframes] ${flag} "${text}" is not a duration; using ${fallback} ms.`);
1617
return fallback;
1718
}
1819
const unit = match[2]?.toLowerCase() as keyof typeof UNIT_MS | undefined;
@@ -21,7 +22,7 @@ export function parseTimeoutMs(
2122
const used = Math.max(ms, floorMs);
2223
if (!unit && ms < 1_000) {
2324
warn(
24-
`[hyperframes] --timeout ${text} is ${ms} ms, so ${used} ms is used. Write ${text}s for seconds.`,
25+
`[hyperframes] ${flag} ${text} is ${ms} ms, so ${used} ms is used. Write ${text}s for seconds.`,
2526
);
2627
}
2728
return used;

0 commit comments

Comments
 (0)