Skip to content

Commit 7173576

Browse files
committed
fix(recording): normalize transformed video bounds before overlays
1 parent e155fbf commit 7173576

4 files changed

Lines changed: 224 additions & 5 deletions

File tree

‎apple/runner/AgentDeviceRunner/RecordingScripts/recording-overlay.swift‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,8 @@ func run() throws {
8585
throw RecordingScriptError.exportFailed("Failed to create composition video track.")
8686
}
8787

88-
let renderSize = resolvedRenderSize(for: sourceVideoTrack)
88+
let sourceBounds = resolvedRenderBounds(for: sourceVideoTrack)
89+
let renderSize = sourceBounds.size
8990
let videoComposition = AVMutableVideoComposition()
9091
videoComposition.renderSize = renderSize
9192
videoComposition.frameDuration = resolvedFrameDuration(
@@ -96,7 +97,10 @@ func run() throws {
9697
let instruction = AVMutableVideoCompositionInstruction()
9798
instruction.timeRange = fullRange
9899
let layerInstruction = AVMutableVideoCompositionLayerInstruction(assetTrack: compositionVideoTrack)
99-
layerInstruction.setTransform(sourceVideoTrack.preferredTransform, at: .zero)
100+
let renderTransform = sourceVideoTrack.preferredTransform.concatenating(
101+
CGAffineTransform(translationX: -sourceBounds.minX, y: -sourceBounds.minY)
102+
)
103+
layerInstruction.setTransform(renderTransform, at: .zero)
100104
instruction.layerInstructions = [layerInstruction]
101105
videoComposition.instructions = [instruction]
102106

@@ -237,8 +241,11 @@ func exportPresetName(compatibleWith asset: AVAsset) throws -> String {
237241
}
238242

239243
func resolvedRenderSize(for track: AVAssetTrack) -> CGSize {
240-
let transformed = track.naturalSize.applying(track.preferredTransform)
241-
return CGSize(width: abs(transformed.width), height: abs(transformed.height))
244+
resolvedRenderBounds(for: track).size
245+
}
246+
247+
func resolvedRenderBounds(for track: AVAssetTrack) -> CGRect {
248+
CGRect(origin: .zero, size: track.naturalSize).applying(track.preferredTransform)
242249
}
243250

244251
/// Guards the compositor's own contract before the caller adopts its output: the burn-in must keep

‎docs/adr/0025-foldable-apple-panels.md‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -379,6 +379,15 @@ The device-lane test reads each export's transform-applied size back through AVF
379379
`resolvedRenderSize` the overlay tool uses — so a `rot90` panel is compared upright rather than on its
380380
sideways coding grid, and no second MP4 geometry parser is added to the shipped packages.
381381

382+
A quarter-turn transform can leave the upright frame outside a zero-origin composition canvas.
383+
The Duo inner capture has a `2006x2852` natural size and transform `[0, 1, -1, 0, 0, 0]`, which
384+
places its upright `2852x2006` bounds at `x=-2852`. Applying that transform without translating the
385+
bounds into the render canvas clips the source content and trips the compositor's all-black check
386+
(#2774). The overlay now derives its render size and origin translation from the same transformed
387+
bounds. Native export regressions cover an identity transform and both translated and
388+
negative-origin quarter turns, asserting upright source content, unchanged geometry, and drawn
389+
touches.
390+
382391
## Verified on a booted Duo
383392

384393
Closed pose: `screenshot` moved from `669x951` luma 0.09 to `466x678` luma 65.8; a tap on Safari's

‎packages/capture-kit/src/recording/__tests__/recording-scripts.test.ts‎

Lines changed: 141 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,15 @@
11
import { beforeAll, describe, test } from 'vitest';
22
import assert from 'node:assert/strict';
3+
import fs from 'node:fs';
34
import path from 'node:path';
45
import { fileURLToPath } from 'node:url';
6+
import { PNG } from 'pngjs';
57
import { runCmd } from '@agent-device/host-kit/command';
68
import { AppError } from '@agent-device/kernel/errors';
79
import { CONTACT_SHEET_UNSUPPORTED_HOST_REASON } from '../contact-sheet-report.ts';
810
import { assertContactSheetHostSupport } from '../contact-sheet-frames.ts';
911
import { getRecordingOverlaySupportWarning } from '../overlay.ts';
12+
import { mkdtempForTest } from '../../tmp-dir.fixtures.ts';
1013

1114
const __dirname = path.dirname(fileURLToPath(import.meta.url));
1215
const recordingScriptsDir = path.resolve(
@@ -24,8 +27,9 @@ type TypecheckOutcome = { exitCode: number; stderr: string; source: string };
2427
// gates in platform-apple keep their native compiles out of test-case wall time the same way.
2528
// One beforeAll covers all three so the SDK probe and the two shared-source compiles are paid once
2629
// per file rather than once per case.
27-
describe.skipIf(process.platform !== 'darwin')('recording Swift scripts typecheck', () => {
30+
describe.skipIf(process.platform !== 'darwin')('recording Swift scripts', () => {
2831
let outcomes: TypecheckOutcome[] = [];
32+
let exports: OverlayExportOutcome[] = [];
2933

3034
beforeAll(
3135
async () => {
@@ -58,6 +62,7 @@ describe.skipIf(process.platform !== 'darwin')('recording Swift scripts typechec
5862
return { source, exitCode: result.exitCode, stderr: result.stderr };
5963
}),
6064
);
65+
exports = await exerciseOverlayExports(swiftCompilerPath, swiftSdkPath, sharedSupport);
6166
},
6267
SWIFT_TYPECHECK_TIMEOUT_MS * 3 + 30_000,
6368
);
@@ -73,8 +78,143 @@ describe.skipIf(process.platform !== 'darwin')('recording Swift scripts typechec
7378
test('recording frames Swift script typechecks', () => {
7479
assertTypechecked(outcomes, 'recording-frames.swift');
7580
});
81+
82+
for (const transform of ['identity', 'rot90-negative-origin', 'rot90-translated']) {
83+
test(`overlay keeps upright content and touches for ${transform}`, () => {
84+
const outcome = exports.find((candidate) => candidate.transform === transform);
85+
assert.ok(outcome);
86+
assert.equal(outcome.exitCode, 0, outcome.stderr);
87+
assert.deepEqual(outcome.outputSize, outcome.sourceSize);
88+
assert.ok(outcome.sourceLuma! > 100);
89+
assert.ok(outcome.outputLuma! > 100, JSON.stringify(outcome));
90+
assert.equal(outcome.sourceBluePixels, 0, 'the fixture must not contain overlay colors');
91+
assert.ok(outcome.outputBluePixels! > 180, 'the touch overlay must actually draw');
92+
assert.ok(outcome.markerDistance! < 3, 'the asymmetric source marker must stay upright');
93+
});
94+
}
7695
});
7796

97+
type OverlayExportOutcome = {
98+
transform: string;
99+
exitCode: number;
100+
stderr: string;
101+
sourceSize?: [number, number];
102+
outputSize?: [number, number];
103+
sourceLuma?: number;
104+
outputLuma?: number;
105+
sourceBluePixels?: number;
106+
outputBluePixels?: number;
107+
markerDistance?: number;
108+
};
109+
110+
async function exerciseOverlayExports(compiler: string, sdk: string, sharedSupport: string) {
111+
const dir = await mkdtempForTest('recording-overlay-export-');
112+
const overlay = path.join(dir, 'overlay');
113+
const fixture = path.join(dir, 'fixture');
114+
const inspect = path.join(dir, 'inspect');
115+
await Promise.all(
116+
[
117+
[overlay, path.join(recordingScriptsDir, 'recording-overlay.swift'), sharedSupport],
118+
[fixture, path.join(recordingTestSupportDir, 'recording-overlay-fixture.swift')],
119+
[inspect, path.join(recordingTestSupportDir, 'recording-inspect.swift')],
120+
].map(async ([output, ...sources]) => {
121+
await runCmd(compiler, ['-sdk', sdk, ...sources, '-o', output!], {
122+
timeoutMs: SWIFT_TYPECHECK_TIMEOUT_MS,
123+
});
124+
}),
125+
);
126+
const results: OverlayExportOutcome[] = [];
127+
for (const transform of ['identity', 'rot90-negative-origin', 'rot90-translated']) {
128+
const source = path.join(dir, `${transform}.mp4`);
129+
const output = path.join(dir, `${transform}-overlay.mp4`);
130+
const events = path.join(dir, `${transform}.json`);
131+
const [width, height] = transform === 'identity' ? [480, 640] : [640, 480];
132+
fs.writeFileSync(
133+
events,
134+
JSON.stringify({
135+
events: [
136+
{
137+
kind: 'tap',
138+
tMs: 150,
139+
x: width! / 2,
140+
y: height! / 2,
141+
referenceWidth: width,
142+
referenceHeight: height,
143+
},
144+
],
145+
}),
146+
);
147+
await runCmd(fixture, [source, transform], { timeoutMs: 15_000 });
148+
const exported = await runCmd(
149+
overlay,
150+
['--input', source, '--output', output, '--events', events],
151+
{ allowFailure: true, timeoutMs: 15_000 },
152+
);
153+
const outcome: OverlayExportOutcome = {
154+
transform,
155+
exitCode: exported.exitCode,
156+
stderr: exported.stderr,
157+
};
158+
if (exported.exitCode === 0) {
159+
const inspected = [];
160+
for (const [label, video] of [
161+
['source', source],
162+
['output', output],
163+
]) {
164+
const inspectDir = path.join(dir, `${transform}-${label}`);
165+
await runCmd(inspect, ['--input', video!, '--events', events, '--output-dir', inspectDir], {
166+
timeoutMs: 15_000,
167+
});
168+
const manifest = JSON.parse(
169+
fs.readFileSync(path.join(inspectDir, 'manifest.json'), 'utf8'),
170+
);
171+
inspected.push({
172+
size: [manifest.renderWidth, manifest.renderHeight] as [number, number],
173+
pixels: inspectExportPixels(manifest.items[0].fullFramePath),
174+
});
175+
}
176+
const [raw, composited] = inspected;
177+
outcome.sourceSize = raw!.size;
178+
outcome.outputSize = composited!.size;
179+
outcome.sourceLuma = raw!.pixels.luma;
180+
outcome.outputLuma = composited!.pixels.luma;
181+
outcome.sourceBluePixels = raw!.pixels.bluePixels;
182+
outcome.outputBluePixels = composited!.pixels.bluePixels;
183+
outcome.markerDistance = Math.hypot(
184+
raw!.pixels.markerX - composited!.pixels.markerX,
185+
raw!.pixels.markerY - composited!.pixels.markerY,
186+
);
187+
}
188+
results.push(outcome);
189+
}
190+
return results;
191+
}
192+
193+
function inspectExportPixels(framePath: string) {
194+
const png = PNG.sync.read(fs.readFileSync(framePath));
195+
let luma = 0;
196+
let bluePixels = 0;
197+
let markerPixels = 0;
198+
let markerX = 0;
199+
let markerY = 0;
200+
for (let i = 0; i < png.width * png.height; i++) {
201+
const [r, g, b] = png.data.subarray(i * 4, i * 4 + 3);
202+
luma += 0.299 * r! + 0.587 * g! + 0.114 * b!;
203+
if (b! >= 180 && g! >= 160 && r! <= 235 && b! >= g! && b! - r! >= 25) bluePixels++;
204+
if (r! > 170 && g! < 100 && b! < 100) {
205+
markerPixels++;
206+
markerX += i % png.width;
207+
markerY += Math.floor(i / png.width);
208+
}
209+
}
210+
return {
211+
luma: luma / (png.width * png.height),
212+
bluePixels,
213+
markerX: markerX / markerPixels,
214+
markerY: markerY / markerPixels,
215+
};
216+
}
217+
78218
function assertTypechecked(outcomes: readonly TypecheckOutcome[], basename: string): void {
79219
const outcome = outcomes.find((entry) => path.basename(entry.source) === basename);
80220
assert.ok(outcome, `${basename} was never typechecked`);
Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
import AVFoundation
2+
import CoreVideo
3+
import Foundation
4+
5+
let outputURL = URL(fileURLWithPath: CommandLine.arguments[1])
6+
let transformName = CommandLine.arguments[2]
7+
let width = 480
8+
let height = 640
9+
let writer = try AVAssetWriter(outputURL: outputURL, fileType: .mp4)
10+
let input = AVAssetWriterInput(mediaType: .video, outputSettings: [
11+
AVVideoCodecKey: AVVideoCodecType.h264,
12+
AVVideoWidthKey: width,
13+
AVVideoHeightKey: height,
14+
])
15+
switch transformName {
16+
case "identity":
17+
input.transform = .identity
18+
case "rot90-negative-origin":
19+
input.transform = CGAffineTransform(a: 0, b: 1, c: -1, d: 0, tx: 0, ty: 0)
20+
case "rot90-translated":
21+
input.transform = CGAffineTransform(a: 0, b: 1, c: -1, d: 0, tx: CGFloat(height), ty: 0)
22+
default:
23+
fatalError("Unknown fixture transform: \(transformName)")
24+
}
25+
let adaptor = AVAssetWriterInputPixelBufferAdaptor(
26+
assetWriterInput: input,
27+
sourcePixelBufferAttributes: [
28+
kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32ARGB,
29+
kCVPixelBufferWidthKey as String: width,
30+
kCVPixelBufferHeightKey as String: height,
31+
]
32+
)
33+
writer.add(input)
34+
guard writer.startWriting() else { fatalError("Writer failed: \(String(describing: writer.error))") }
35+
writer.startSession(atSourceTime: .zero)
36+
for frame in 0..<36 {
37+
while !input.isReadyForMoreMediaData { Thread.sleep(forTimeInterval: 0.002) }
38+
var buffer: CVPixelBuffer?
39+
guard CVPixelBufferPoolCreatePixelBuffer(nil, adaptor.pixelBufferPool!, &buffer) == kCVReturnSuccess,
40+
let buffer
41+
else { fatalError("Could not allocate fixture frame") }
42+
CVPixelBufferLockBaseAddress(buffer, [])
43+
let base = CVPixelBufferGetBaseAddress(buffer)!.assumingMemoryBound(to: UInt8.self)
44+
let stride = CVPixelBufferGetBytesPerRow(buffer)
45+
for y in 0..<height {
46+
for x in 0..<width {
47+
let offset = y * stride + x * 4
48+
let redMarker = x < 15 && y < 25
49+
base[offset] = 255
50+
base[offset + 1] = 245
51+
base[offset + 2] = redMarker ? 30 : 245
52+
base[offset + 3] = redMarker ? 30 : 245
53+
}
54+
}
55+
CVPixelBufferUnlockBaseAddress(buffer, [])
56+
guard adaptor.append(buffer, withPresentationTime: CMTime(value: Int64(frame), timescale: 30))
57+
else { fatalError("Append failed: \(String(describing: writer.error))") }
58+
}
59+
input.markAsFinished()
60+
let finished = DispatchSemaphore(value: 0)
61+
writer.finishWriting { finished.signal() }
62+
guard finished.wait(timeout: .now() + 10) == .success, writer.status == .completed
63+
else { fatalError("Finish failed: \(String(describing: writer.error))") }

0 commit comments

Comments
 (0)