Skip to content

Commit 39ad272

Browse files
authored
fix(check): keep auditing when a motion spec is partly unusable (#2805)
Combines two fixes to the motion sidecar path: - An ambiguous selector no longer aborts the whole spec. It is reported as a finding, the assertions that depend on it are skipped and named, and every other assertion still evaluates. `assertionTargets` is now exhaustive over the assertion kinds so a new kind cannot silently fall back to an arbitrary first match. - A sidecar that will not parse is reported as a spec finding instead of ending the run, so the composition is still audited.
1 parent 7a2a691 commit 39ad272

3 files changed

Lines changed: 230 additions & 33 deletions

File tree

‎packages/cli/src/commands/check.test.ts‎

Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ import {
3232
} from "../utils/checkPipeline.js";
3333
import { resolveCompositionViewportFromHtml } from "../utils/compositionViewport.js";
3434
import { consumeCommandResult } from "../utils/commandResult.js";
35+
import { ambiguousIssue } from "../utils/motionAudit.js";
3536
import type { ProjectLintResult } from "../utils/lintProject.js";
3637
import type {
3738
LayoutIssue,
@@ -1110,6 +1111,171 @@ describe("check pipeline", () => {
11101111
expect(browser).toHaveBeenCalledTimes(1);
11111112
});
11121113

1114+
it("still audits layout when the motion sidecar is invalid", async () => {
1115+
const motion: MotionSpecResolution = {
1116+
kind: "invalid",
1117+
path: "/project/index.motion.json",
1118+
message: 'assertions[0] (staysInFrame): "selector" must be a non-empty string',
1119+
};
1120+
const driver = fakeDriver({
1121+
collectLayout: vi.fn(async (time: number) => [layoutIssue("error", { time })]),
1122+
});
1123+
const { report, browser } = await runScenario(driver, {}, { motion });
1124+
1125+
expect(browser).toHaveBeenCalledTimes(1);
1126+
expect(report.layout.findings.length).toBeGreaterThan(0);
1127+
expect(report.motion.findings).toEqual([
1128+
expect.objectContaining({ code: "motion_spec_invalid", severity: "error" }),
1129+
]);
1130+
expect(report.motion).toMatchObject({
1131+
enabled: true,
1132+
specPath: "/project/index.motion.json",
1133+
samples: 0,
1134+
});
1135+
expect(report.ok).toBe(false);
1136+
});
1137+
1138+
it("does not sample motion for an invalid sidecar", async () => {
1139+
const motion: MotionSpecResolution = {
1140+
kind: "invalid",
1141+
path: "/project/index.motion.json",
1142+
message: "version 2 is not supported",
1143+
};
1144+
const driver = fakeDriver();
1145+
const { report } = await runScenario(driver, {}, { motion });
1146+
1147+
expect(report.motion.samples).toBe(0);
1148+
expect(driver.collectMotionFrame).not.toHaveBeenCalled();
1149+
});
1150+
1151+
it("keeps evaluating the unambiguous assertions when one selector is ambiguous", async () => {
1152+
const motion: MotionSpecResolution = {
1153+
kind: "valid",
1154+
path: "/project/index.motion.json",
1155+
spec: {
1156+
assertions: [
1157+
{ kind: "appearsBy", selector: ".item", bySec: 1 },
1158+
{ kind: "appearsBy", selector: "#hero", bySec: 0.2 },
1159+
],
1160+
},
1161+
};
1162+
const driver = fakeDriver({
1163+
getDuration: vi.fn(async () => 1),
1164+
findAmbiguousSelectors: vi.fn(async () => [
1165+
{ ...layoutIssue("error", { code: "motion_selector_ambiguous" }), selector: ".item" },
1166+
]),
1167+
collectMotionFrame: vi.fn(async (time: number) => heroMotionFrame(time, (t) => t >= 0.5)),
1168+
});
1169+
const { report } = await runScenario(driver, {}, { motion });
1170+
1171+
expect(report.motion.samples).toBeGreaterThan(0);
1172+
expect(report.motion.findings.map((finding) => finding.code).sort()).toEqual([
1173+
"motion_appears_late",
1174+
"motion_selector_ambiguous",
1175+
]);
1176+
expect(report.ok).toBe(false);
1177+
expect(
1178+
report.motion.findings.find((finding) => finding.code === "motion_selector_ambiguous")
1179+
?.message,
1180+
).toContain("1 assertion(s) naming it were not evaluated");
1181+
expect(driver.collectMotionFrame).toHaveBeenCalledWith(expect.any(Number), ["#hero"], []);
1182+
});
1183+
1184+
it("separates the ambiguity message from the skipped-assertion count", async () => {
1185+
const motion: MotionSpecResolution = {
1186+
kind: "valid",
1187+
path: "/project/index.motion.json",
1188+
spec: { assertions: [{ kind: "appearsBy", selector: ".item", bySec: 1 }] },
1189+
};
1190+
const driver = fakeDriver({
1191+
getDuration: vi.fn(async () => 1),
1192+
findAmbiguousSelectors: vi.fn(async () => [
1193+
{ ...layoutIssue("error"), ...ambiguousIssue(".item") },
1194+
]),
1195+
});
1196+
const { report } = await runScenario(driver, {}, { motion });
1197+
1198+
const message = report.motion.findings.find(
1199+
(finding) => finding.code === "motion_selector_ambiguous",
1200+
)?.message;
1201+
expect(message).toContain("exactly one. 1 assertion(s)");
1202+
});
1203+
1204+
it("counts skipped assertions per ambiguous selector, not across all of them", async () => {
1205+
const motion: MotionSpecResolution = {
1206+
kind: "valid",
1207+
path: "/project/index.motion.json",
1208+
spec: {
1209+
assertions: [
1210+
{ kind: "appearsBy", selector: ".item", bySec: 1 },
1211+
{ kind: "appearsBy", selector: ".card", bySec: 1 },
1212+
{ kind: "appearsBy", selector: ".card", bySec: 2 },
1213+
],
1214+
},
1215+
};
1216+
const driver = fakeDriver({
1217+
getDuration: vi.fn(async () => 1),
1218+
findAmbiguousSelectors: vi.fn(async () => [
1219+
{ ...layoutIssue("error", { code: "motion_selector_ambiguous" }), selector: ".item" },
1220+
{ ...layoutIssue("error", { code: "motion_selector_ambiguous" }), selector: ".card" },
1221+
]),
1222+
});
1223+
const { report } = await runScenario(driver, {}, { motion });
1224+
1225+
const messages = report.motion.findings
1226+
.filter((finding) => finding.code === "motion_selector_ambiguous")
1227+
.map((finding) => finding.message);
1228+
expect(messages.filter((m) => m.includes("1 assertion(s) naming it"))).toHaveLength(1);
1229+
expect(messages.filter((m) => m.includes("2 assertion(s) naming it"))).toHaveLength(1);
1230+
});
1231+
1232+
it("drops a before assertion when either side is ambiguous, keeping the rest", async () => {
1233+
const motion: MotionSpecResolution = {
1234+
kind: "valid",
1235+
path: "/project/index.motion.json",
1236+
spec: {
1237+
assertions: [
1238+
{ kind: "before", a: "#hero", b: ".item" },
1239+
{ kind: "appearsBy", selector: "#hero", bySec: 0.2 },
1240+
],
1241+
},
1242+
};
1243+
const driver = fakeDriver({
1244+
getDuration: vi.fn(async () => 1),
1245+
findAmbiguousSelectors: vi.fn(async () => [
1246+
{ ...layoutIssue("error", { code: "motion_selector_ambiguous" }), selector: ".item" },
1247+
]),
1248+
collectMotionFrame: vi.fn(async (time: number) => heroMotionFrame(time, (t) => t >= 0.5)),
1249+
});
1250+
const { report } = await runScenario(driver, {}, { motion });
1251+
1252+
expect(report.motion.findings.map((finding) => finding.code).sort()).toEqual([
1253+
"motion_appears_late",
1254+
"motion_selector_ambiguous",
1255+
]);
1256+
expect(driver.collectMotionFrame).toHaveBeenCalledWith(expect.any(Number), ["#hero"], []);
1257+
});
1258+
1259+
it("does not sample when every assertion names an ambiguous selector", async () => {
1260+
const motion: MotionSpecResolution = {
1261+
kind: "valid",
1262+
path: "/project/index.motion.json",
1263+
spec: { assertions: [{ kind: "appearsBy", selector: ".item", bySec: 1 }] },
1264+
};
1265+
const driver = fakeDriver({
1266+
getDuration: vi.fn(async () => 1),
1267+
findAmbiguousSelectors: vi.fn(async () => [
1268+
{ ...layoutIssue("error", { code: "motion_selector_ambiguous" }), selector: ".item" },
1269+
]),
1270+
});
1271+
const { report } = await runScenario(driver, {}, { motion });
1272+
1273+
expect(report.motion.samples).toBe(0);
1274+
expect(report.motion.findings.map((finding) => finding.code)).toEqual([
1275+
"motion_selector_ambiguous",
1276+
]);
1277+
});
1278+
11131279
it("reports a failing appearsBy sidecar as motion_appears_late", async () => {
11141280
const motion: MotionSpecResolution = {
11151281
kind: "valid",

‎packages/cli/src/utils/checkPipeline.ts‎

Lines changed: 40 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,13 @@ import {
1616
type LayoutRect,
1717
} from "./layoutAudit.js";
1818
import {
19+
assertionTargets,
1920
collectSamplingTargets,
2021
evaluateMotion,
2122
type Canvas,
2223
type MotionFrame,
2324
} from "./motionAudit.js";
24-
import { findMotionSpec, readMotionSpec } from "./motionSpec.js";
25+
import { findMotionSpec, readMotionSpec, type MotionAssertion } from "./motionSpec.js";
2526
import { normalizeErrorMessage } from "./errorMessage.js";
2627
import {
2728
parseColorRGBA,
@@ -169,6 +170,7 @@ interface MotionPlan {
169170
selectors: string[];
170171
livenessScopes: string[];
171172
preflightIssues: AnchoredLayoutIssue[];
173+
assertions: MotionAssertion[];
172174
}
173175

174176
async function planMotionSampling(
@@ -177,13 +179,30 @@ async function planMotionSampling(
177179
duration: number,
178180
): Promise<MotionPlan> {
179181
if (motion.kind !== "valid") {
180-
return { times: [], selectors: [], livenessScopes: [], preflightIssues: [] };
182+
return { times: [], selectors: [], livenessScopes: [], preflightIssues: [], assertions: [] };
181183
}
182-
const targets = collectSamplingTargets(motion.spec.assertions);
183-
const preflightIssues = await driver.findAmbiguousSelectors(targets.selectors);
184+
const preflightIssues = await driver.findAmbiguousSelectors(
185+
collectSamplingTargets(motion.spec.assertions).selectors,
186+
);
187+
const ambiguous = new Set(preflightIssues.map((issue) => issue.selector));
188+
const assertions = motion.spec.assertions.filter(
189+
(assertion) =>
190+
!assertionTargets(assertion).selectors.some((selector) => ambiguous.has(selector)),
191+
);
192+
noteSkippedAssertions(preflightIssues, motion.spec.assertions);
193+
const targets = collectSamplingTargets(assertions);
184194
const times =
185-
preflightIssues.length === 0 ? buildMotionSampleTimes(motion.spec.duration ?? duration) : [];
186-
return { times, ...targets, preflightIssues };
195+
assertions.length > 0 ? buildMotionSampleTimes(motion.spec.duration ?? duration) : [];
196+
return { times, ...targets, preflightIssues, assertions };
197+
}
198+
199+
function noteSkippedAssertions(issues: AnchoredLayoutIssue[], assertions: MotionAssertion[]): void {
200+
for (const issue of issues) {
201+
const skipped = assertions.filter((assertion) =>
202+
assertionTargets(assertion).selectors.includes(issue.selector),
203+
).length;
204+
if (skipped > 0) issue.message += ` ${skipped} assertion(s) naming it were not evaluated.`;
205+
}
187206
}
188207

189208
interface GridSamples {
@@ -1061,13 +1080,13 @@ export async function runAuditGrid(
10611080
const seekLoopMs = Date.now() - seekLoopStart;
10621081

10631082
let motionIssues = plan.preflightIssues;
1064-
if (motion.kind === "valid" && motionIssues.length === 0 && collected.motionFrames.length > 0) {
1083+
if (motion.kind === "valid" && plan.assertions.length > 0 && collected.motionFrames.length > 0) {
10651084
const evaluated = evaluateMotion(
10661085
collected.motionFrames,
1067-
motion.spec.assertions,
1086+
plan.assertions,
10681087
await driver.getCanvas(),
10691088
);
1070-
motionIssues = await driver.anchorMotionIssues(evaluated);
1089+
motionIssues = [...motionIssues, ...(await driver.anchorMotionIssues(evaluated))];
10711090
}
10721091
const sweepFindings = detectSweepStatic(
10731092
grid.duration,
@@ -1131,15 +1150,17 @@ export async function runCheckPipeline(
11311150
}
11321151

11331152
const motion = dependencies.resolveMotionSpec(project.dir);
1134-
if (motion.kind === "invalid") {
1135-
const finding = findingAtRoot(
1136-
"motion_spec_invalid",
1137-
"error",
1138-
motion.message,
1139-
relative(project.dir, motion.path) || "index.motion.json",
1140-
);
1141-
return buildReport(options, lint, emptyBrowserResult(), motion, [finding], []);
1142-
}
1153+
const specFindings =
1154+
motion.kind === "invalid"
1155+
? [
1156+
findingAtRoot(
1157+
"motion_spec_invalid",
1158+
"error",
1159+
motion.message,
1160+
relative(project.dir, motion.path) || "index.motion.json",
1161+
),
1162+
]
1163+
: [];
11431164

11441165
let browser: CheckBrowserResult;
11451166
try {
@@ -1152,7 +1173,7 @@ export async function runCheckPipeline(
11521173
const snapshotFiles = options.snapshots
11531174
? await writeContrastSnapshots(dependencies, project.dir, browser)
11541175
: [];
1155-
const report = buildReport(options, lint, browser, motion, [], snapshotFiles);
1176+
const report = buildReport(options, lint, browser, motion, specFindings, snapshotFiles);
11561177
return options.snapshots
11571178
? await withFindingCrops(dependencies, project, options, report)
11581179
: report;

‎packages/cli/src/utils/motionAudit.ts‎

Lines changed: 24 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ export function ambiguousIssue(selector: string): LayoutIssue {
3636
severity: "error",
3737
time: 0,
3838
selector,
39-
message: `${selector} matches multiple elements — use a more specific selector so the assertion targets exactly one`,
39+
message: `${selector} matches multiple elements — use a more specific selector so the assertion targets exactly one.`,
4040
rect: ZERO_RECT,
4141
fixHint:
4242
"Use #id or :nth-child() instead of a class selector when multiple elements share the same class.",
@@ -236,6 +236,26 @@ export function evaluateMotion(
236236
});
237237
}
238238

239+
export function assertionTargets(assertion: MotionAssertion): {
240+
selectors: string[];
241+
scopes: string[];
242+
} {
243+
switch (assertion.kind) {
244+
case "appearsBy":
245+
case "staysInFrame":
246+
return { selectors: [assertion.selector], scopes: [] };
247+
case "before":
248+
return { selectors: [assertion.a, assertion.b], scopes: [] };
249+
case "keepsMoving":
250+
return { selectors: [], scopes: [assertion.withinSelector ?? "*"] };
251+
default: {
252+
const exhaustive: never = assertion;
253+
void exhaustive;
254+
return { selectors: [], scopes: [] };
255+
}
256+
}
257+
}
258+
239259
/**
240260
* Selectors and liveness scopes the in-page sampler must read for a spec.
241261
* Selectors feed the per-element matrix; scopes feed keepsMoving liveness.
@@ -247,19 +267,9 @@ export function collectSamplingTargets(assertions: MotionAssertion[]): {
247267
const selectors = new Set<string>();
248268
const scopes = new Set<string>();
249269
for (const assertion of assertions) {
250-
switch (assertion.kind) {
251-
case "appearsBy":
252-
case "staysInFrame":
253-
selectors.add(assertion.selector);
254-
break;
255-
case "before":
256-
selectors.add(assertion.a);
257-
selectors.add(assertion.b);
258-
break;
259-
case "keepsMoving":
260-
scopes.add(assertion.withinSelector ?? "*");
261-
break;
262-
}
270+
const targets = assertionTargets(assertion);
271+
for (const selector of targets.selectors) selectors.add(selector);
272+
for (const scope of targets.scopes) scopes.add(scope);
263273
}
264274
return { selectors: [...selectors], livenessScopes: [...scopes] };
265275
}

0 commit comments

Comments
 (0)