Skip to content

Commit a81b344

Browse files
feat(cli): a line_unread finding carries the frames it measured around where the line was first read (heygen-com#7294)
* feat(cli): a line_unread finding carries the frames it measured around where the line was first read * style(cli): shorten the measured field's comment * feat(cli): measured frames at both ends of the line, with fewer comment lines
1 parent b4cc674 commit a81b344

3 files changed

Lines changed: 76 additions & 2 deletions

File tree

‎packages/cli/src/utils/checkTypes.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -350,6 +350,7 @@ export interface CraftFinding extends CheckFinding {
350350
fontPx?: number;
351351
floorPx?: number;
352352
until?: number;
353+
measured?: { time: number; opacity: number | null; heightPx: number | null }[];
353354
elements?: { selector: string; text: string; fontPx: number; time: number; seconds: number }[];
354355
}
355356

‎packages/cli/src/utils/craftAudit.test.ts‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,34 @@ describe("line_unread", () => {
119119
expect(finding?.message).toContain("references/motion.md");
120120
});
121121

122+
// fails without the fix: a finding carried no measured frames, so a report of a line read late could not show why.
123+
it("carries the frames it measured at both ends of the line, with none painted before or after it", async () => {
124+
const shown: Shown[] = [
125+
{ key: 1, text: "Every team gets one shared calendar now", from: 6.2, to: 6.9 },
126+
{ key: 2, text: "Try it today", from: 13, to: 20 },
127+
];
128+
const [finding] = await detectLineUnread(film(20, shown), 20, CANVAS, probeFor(shown));
129+
const measured = finding?.measured ?? [];
130+
expect(measured.map((frame) => frame.opacity)).toEqual([
131+
null,
132+
null,
133+
null,
134+
1,
135+
1,
136+
1,
137+
1,
138+
1,
139+
1,
140+
null,
141+
null,
142+
null,
143+
]);
144+
expect(measured[3]).toMatchObject({ time: 6.25, heightPx: 115 });
145+
expect(measured.map((frame) => frame.time)).toEqual([
146+
5.875, 6, 6.125, 6.25, 6.375, 6.5, 6.625, 6.75, 6.875, 7, 7.125, 7.25,
147+
]);
148+
});
149+
122150
it("does not warn on a landed title held 1.5 s, a 0.4 s chapter word, captions or a 2.5 s end card", async () => {
123151
const shown: Shown[] = [
124152
{ key: 1, text: "Ship it faster", from: 0.5, to: 2 },

‎packages/cli/src/utils/craftAudit.ts‎

Lines changed: 47 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -301,6 +301,7 @@ interface LineCandidate {
301301
floor: number;
302302
window: LineWindow;
303303
firstIndex: number;
304+
lastIndex: number;
304305
}
305306

306307
/** A run that reaches the last sample lasts to the film's end; one at sample 0 starts exactly there. */
@@ -348,7 +349,16 @@ function lineCandidates(frames: CraftFrame[], duration: number, canvas: Canvas):
348349
const floor = lineFloorSec(words, finalLine);
349350
// Clearly long enough even at the shortest the grid allows.
350351
if (window.endIn - window.startIn >= floor) continue;
351-
candidates.push({ track, words, text, finalLine, floor, window, firstIndex: run[0] });
352+
candidates.push({
353+
track,
354+
words,
355+
text,
356+
finalLine,
357+
floor,
358+
window,
359+
firstIndex: run[0],
360+
lastIndex: run[1],
361+
});
352362
}
353363
return candidates;
354364
}
@@ -393,7 +403,7 @@ export async function detectLineUnread(
393403
const held = windowSeconds(window);
394404
if (held >= candidate.floor - T.measureToleranceSec) continue;
395405
const error = Math.max(window.startIn - window.startOut, window.endOut - window.endIn) / 2;
396-
findings.push(lineUnreadFinding(candidate, window, held, error));
406+
findings.push(lineUnreadFinding(candidate, window, held, error, frames));
397407
}
398408
return findings;
399409
}
@@ -403,6 +413,7 @@ function lineUnreadFinding(
403413
window: LineWindow,
404414
held: number,
405415
error: number,
416+
frames: CraftFrame[],
406417
): CraftFinding {
407418
const { track, words, text, finalLine, floor } = candidate;
408419
const quoted = quote(text, 60);
@@ -429,9 +440,43 @@ function lineUnreadFinding(
429440
words,
430441
seconds: round(held),
431442
floorSeconds: round(floor),
443+
measured: measuredAround(frames, track.key, candidate.firstIndex, candidate.lastIndex),
432444
};
433445
}
434446

447+
const MEASURED_SPAN = 3;
448+
449+
/** The samples at both ends of the line's run, as the sampler measured them. */
450+
function measuredAround(
451+
frames: CraftFrame[],
452+
key: number,
453+
first: number,
454+
last: number,
455+
): NonNullable<CraftFinding["measured"]> {
456+
const indices = new Set<number>();
457+
for (const edge of [first, last]) {
458+
for (let at = edge - MEASURED_SPAN; at <= edge + MEASURED_SPAN; at += 1) indices.add(at);
459+
}
460+
const out: NonNullable<CraftFinding["measured"]> = [];
461+
for (const index of [...indices].sort((a, b) => a - b)) {
462+
const frame = frames[index];
463+
if (!frame) continue;
464+
const parts = frame.parts.filter((part) => part.block === key && !part.captions);
465+
if (parts.length === 0) {
466+
out.push({ time: round(frame.time), opacity: null, heightPx: null });
467+
continue;
468+
}
469+
const top = Math.min(...parts.map((part) => part.rect.top));
470+
const bottom = Math.max(...parts.map((part) => part.rect.top + part.rect.height));
471+
out.push({
472+
time: round(frame.time),
473+
opacity: round(Math.max(...parts.map((part) => part.opacity))),
474+
heightPx: Math.round(bottom - top),
475+
});
476+
}
477+
return out;
478+
}
479+
435480
// ---------- text_below_floor ----------
436481

437482
interface SmallText {

0 commit comments

Comments
 (0)