Skip to content

Commit 92b8fae

Browse files
devthymikee
authored andcommitted
fix(recording): export touch overlay at capture size, never black (#2707)
The overlay burn-in re-encoded through AVAssetExportPresetMediumQuality, a fixed-canvas preset that caps the long edge at 480px, collapsing any touch recording to ~220x480 (480x220 landscape) and going all-black on long clips, always with exit 0. --hide-touches and an empty gesture list copy the raw capture through, so only the composited path was affected. Both quality tiers now export through the one geometry-preserving preset (HighestQuality); the hardware encoder makes the full-resolution re-encode cheap. The compositor now verifies its own output against the raw before the caller adopts it — resolved track size must match, and a uniformly black track whose raw had visible content is rejected — and throws on failure so the overlay is dropped, the raw kept, and the choice reported as overlayWarning rather than a broken file. Adds an mp4 transform-aware track-size reader and a device-lane assertion that a touched export equals its --hide-touches control.
1 parent b38b1f9 commit 92b8fae

9 files changed

Lines changed: 388 additions & 50 deletions

File tree

‎apple/runner/AgentDeviceRunner/RecordingScripts/recording-overlay.swift‎

Lines changed: 145 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,13 @@ let minimumPinchVisibility: CFTimeInterval = 0.5
1111
let swipeVisibilityTail: CFTimeInterval = 0.16
1212
let trailOpacityKeyTimes: [NSNumber] = [0.0, 0.08, 0.62, 1.0]
1313

14+
// A compositor that cannot render frames still completes with an empty track, so a whole frame at
15+
// or below this mean luma counts as black.
16+
let compositedBlackFrameLuma: Double = 1.0
17+
// Above this mean luma the raw capture is taken to hold visible content worth preserving. Below it
18+
// the recording itself is dark, so a dark composite is the content and not a compositor failure.
19+
let sourceVisibleFrameLuma: Double = 2.0
20+
1421
struct GestureEnvelope: Decodable {
1522
let events: [GestureEvent]
1623
}
@@ -126,10 +133,10 @@ func run() throws {
126133
in: parentLayer
127134
)
128135

129-
// Overlay burn-in forces a full re-encode; medium quality keeps simulator videos readable
130-
// while avoiding very slow highest-quality exports. Pass --quality high to opt into
131-
// the slower highest-quality export.
132-
let presetName = exportPresetName(for: parsedArgs.exportQuality, compatibleWith: composition)
136+
// Overlay burn-in forces a full re-encode. The export has to keep the captured track's own
137+
// dimensions, so it uses the one preset that preserves arbitrary capture geometry; the hardware
138+
// encoder makes the full-resolution re-encode cheap (measured ~1-2s for a 90s 1206x2622 clip).
139+
let presetName = try exportPresetName(compatibleWith: composition)
133140
let exporter = try makeRecordingExporter(
134141
composition,
135142
presetName: presetName,
@@ -141,17 +148,19 @@ func run() throws {
141148
timeoutMessage: "Touch overlay export timed out.",
142149
failureMessage: "Touch overlay export failed."
143150
)
151+
try verifyCompositedOverlay(
152+
input: inputURL,
153+
output: outputURL,
154+
expectedRenderSize: renderSize
155+
)
144156
}
145157

146158
func parseArguments(
147159
_ arguments: [String]
148-
) throws -> (inputPath: String, outputPath: String, eventsPath: String, exportQuality: ExportQuality) {
160+
) throws -> (inputPath: String, outputPath: String, eventsPath: String) {
149161
var inputPath: String?
150162
var outputPath: String?
151163
var eventsPath: String?
152-
// Export quality defaults to medium so existing callers keep the fast, simulator-friendly
153-
// export. Pass --quality high to opt into a slower highest-quality export.
154-
var exportQuality: ExportQuality = .medium
155164
var index = 0
156165

157166
while index < arguments.count {
@@ -168,11 +177,13 @@ func parseArguments(
168177
eventsPath = try recordingOptionValue(arguments, nextIndex, "--events")
169178
index += 2
170179
case "--quality":
180+
// Accepted for CLI parity with the other backends and still validated, but the composited
181+
// export always preserves the captured geometry (see `exportPresetName`), so the tier never
182+
// picks a resolution-capping preset and is not retained.
171183
let rawValue = try recordingOptionValue(arguments, nextIndex, "--quality")
172-
guard let parsed = ExportQuality(rawValue: rawValue) else {
184+
guard ExportQuality(rawValue: rawValue) != nil else {
173185
throw RecordingScriptError.invalidArgs("--quality must be one of: medium, high")
174186
}
175-
exportQuality = parsed
176187
index += 2
177188
default:
178189
throw RecordingScriptError.invalidArgs("Unknown argument: \(argument)")
@@ -184,31 +195,140 @@ func parseArguments(
184195
"Usage: recording-overlay.swift --input <video> --output <video> --events <json> [--quality <medium|high>]"
185196
)
186197
}
187-
return (inputPath, outputPath, eventsPath, exportQuality)
198+
return (inputPath, outputPath, eventsPath)
188199
}
189200

190-
func exportPresetName(
191-
for exportQuality: ExportQuality,
192-
compatibleWith asset: AVAsset
193-
) -> String {
194-
switch exportQuality {
195-
case .high:
196-
return AVAssetExportPresetHighestQuality
197-
case .medium:
198-
// Prefer the faster medium preset, falling back to highest quality only when medium is
199-
// not available for this composition.
200-
let compatible = AVAssetExportSession.exportPresets(compatibleWith: asset)
201-
return compatible.contains(AVAssetExportPresetMediumQuality)
202-
? AVAssetExportPresetMediumQuality
203-
: AVAssetExportPresetHighestQuality
201+
/// The composited overlay must keep the captured track's dimensions, so it can only use a preset
202+
/// that preserves source geometry. The fixed-canvas presets rescale the long edge —
203+
/// `AVAssetExportPresetMediumQuality` caps it at 480px, which is exactly what collapsed every
204+
/// touch-bearing recording to ~220x480 (#2707). Only `HighestQuality` preserves the capture, and it
205+
/// is the export for every quality tier, so `--quality` never trades capture resolution away. If a
206+
/// composition offers no geometry-preserving preset the export refuses rather than rescaling.
207+
func exportPresetName(compatibleWith asset: AVAsset) throws -> String {
208+
guard AVAssetExportSession.exportPresets(compatibleWith: asset).contains(AVAssetExportPresetHighestQuality) else {
209+
throw RecordingScriptError.exportFailed(
210+
"No geometry-preserving export preset is available; refusing to rescale the capture."
211+
)
204212
}
213+
return AVAssetExportPresetHighestQuality
205214
}
206215

207216
func resolvedRenderSize(for track: AVAssetTrack) -> CGSize {
208217
let transformed = track.naturalSize.applying(track.preferredTransform)
209218
return CGSize(width: abs(transformed.width), height: abs(transformed.height))
210219
}
211220

221+
/// Guards the compositor's own contract before the caller adopts its output: the burn-in must keep
222+
/// the captured track's dimensions, must produce frames, and must not silently become an all-black
223+
/// track. Any failure throws, dropping the overlay and keeping the raw capture rather than
224+
/// publishing a broken file with a success exit.
225+
func verifyCompositedOverlay(input: URL, output: URL, expectedRenderSize: CGSize) throws {
226+
let composited = AVURLAsset(url: output)
227+
guard let track = composited.tracks(withMediaType: .video).first else {
228+
throw RecordingScriptError.exportFailed("Touch overlay export produced no video track.")
229+
}
230+
231+
let producedSize = resolvedRenderSize(for: track)
232+
if !renderSizeMatches(producedSize, expectedRenderSize) {
233+
throw RecordingScriptError.exportFailed(
234+
"Touch overlay export changed the track geometry: \(Int(producedSize.width))x\(Int(producedSize.height)) instead of \(Int(expectedRenderSize.width))x\(Int(expectedRenderSize.height))."
235+
)
236+
}
237+
238+
// A compositor that produced no decodable frames is a failure, not an absence of evidence.
239+
guard let compositedLuma = meanFrameLuma(of: composited) else {
240+
throw RecordingScriptError.exportFailed("Touch overlay export produced no decodable frames.")
241+
}
242+
243+
let source = AVURLAsset(url: input)
244+
let sourceDuration = CMTimeGetSeconds(source.duration)
245+
let producedDuration = CMTimeGetSeconds(composited.duration)
246+
if sourceDuration.isFinite, sourceDuration > 1, producedDuration.isFinite,
247+
producedDuration < sourceDuration * 0.5
248+
{
249+
throw RecordingScriptError.exportFailed(
250+
"Touch overlay export truncated the track to \(Int(producedDuration))s of \(Int(sourceDuration))s."
251+
)
252+
}
253+
254+
if compositedLuma <= compositedBlackFrameLuma,
255+
let sourceLuma = meanFrameLuma(of: source),
256+
sourceLuma > sourceVisibleFrameLuma
257+
{
258+
throw RecordingScriptError.exportFailed(
259+
"Touch overlay export produced an all-black track while the raw capture had visible content."
260+
)
261+
}
262+
}
263+
264+
func renderSizeMatches(_ produced: CGSize, _ expected: CGSize) -> Bool {
265+
// The encoder rounds the stored frame to its coding-block grid, so an honest match tolerates a
266+
// few pixels regardless of size. A geometry collapse (the #2707 failure) misses by hundreds.
267+
let tolerance: CGFloat = 32
268+
return abs(produced.width - expected.width) <= tolerance
269+
&& abs(produced.height - expected.height) <= tolerance
270+
}
271+
272+
func meanFrameLuma(of asset: AVURLAsset, sampleCount: Int = 5) -> Double? {
273+
let duration = CMTimeGetSeconds(asset.duration)
274+
guard duration.isFinite, duration > 0 else { return nil }
275+
let generator = AVAssetImageGenerator(asset: asset)
276+
generator.appliesPreferredTrackTransform = true
277+
generator.requestedTimeToleranceBefore = .positiveInfinity
278+
generator.requestedTimeToleranceAfter = .positiveInfinity
279+
// Only the mean brightness is wanted, so decode a thumbnail rather than the full frame.
280+
generator.maximumSize = CGSize(width: 64, height: 64)
281+
282+
var total = 0.0
283+
var sampled = 0
284+
for index in 0..<sampleCount {
285+
let fraction = (Double(index) + 0.5) / Double(sampleCount)
286+
let time = CMTime(seconds: fraction * duration, preferredTimescale: 600)
287+
guard let image = try? generator.copyCGImage(at: time, actualTime: nil),
288+
let luma = frameMeanLuma(image)
289+
else { continue }
290+
total += luma
291+
sampled += 1
292+
}
293+
guard sampled > 0 else { return nil }
294+
return total / Double(sampled)
295+
}
296+
297+
func frameMeanLuma(_ image: CGImage) -> Double? {
298+
let width = image.width
299+
let height = image.height
300+
guard width > 0, height > 0 else { return nil }
301+
let bytesPerRow = width * 4
302+
var pixels = [UInt8](repeating: 0, count: bytesPerRow * height)
303+
let drawn = pixels.withUnsafeMutableBytes { raw -> Bool in
304+
guard
305+
let context = CGContext(
306+
data: raw.baseAddress,
307+
width: width,
308+
height: height,
309+
bitsPerComponent: 8,
310+
bytesPerRow: bytesPerRow,
311+
space: CGColorSpaceCreateDeviceRGB(),
312+
bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue
313+
)
314+
else { return false }
315+
context.draw(image, in: CGRect(x: 0, y: 0, width: width, height: height))
316+
return true
317+
}
318+
guard drawn else { return nil }
319+
320+
var total = 0.0
321+
var count = 0
322+
for offset in stride(from: 0, to: bytesPerRow * height, by: 4) {
323+
total += 0.299 * Double(pixels[offset])
324+
+ 0.587 * Double(pixels[offset + 1])
325+
+ 0.114 * Double(pixels[offset + 2])
326+
count += 1
327+
}
328+
guard count > 0 else { return nil }
329+
return total / Double(count)
330+
}
331+
212332
func resolvedFrameDuration(for track: AVAssetTrack) -> CMTime {
213333
let minFrameDuration = track.minFrameDuration
214334
if minFrameDuration.isValid && !minFrameDuration.isIndefinite && minFrameDuration.seconds > 0 {

‎docs/adr/0025-foldable-apple-panels.md‎

Lines changed: 53 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,9 @@
22

33
## Status
44

5-
Accepted (2026-09-20; the pose-settle rule amended 2026-09-21 under #2730). Covers iPhone Duo
6-
(iOS 27.1, `iPhone19,4`) and any Apple device that reports more than one integrated CoreDevice
7-
display.
5+
Accepted (2026-09-20; the pose-settle rule amended 2026-09-21 under #2730; the touch-overlay
6+
export diagnosis corrected 2026-09-22 under #2707). Covers iPhone Duo (iOS 27.1, `iPhone19,4`) and
7+
any Apple device that reports more than one integrated CoreDevice display.
88

99
An iPhone Duo carries two integrated panels — Apple's **outer display** and **inner display** —
1010
and lights one of them at a time. Which one is lit is the device pose. Two independent facts
@@ -307,18 +307,56 @@ and honors it per panel; sampled mean luma over the whole frame:
307307
| no `--display` | 2006x2852 | 241.42 |
308308

309309
`record start`/`record stop` exit 0 in both poses, and with `--hide-touches` the export keeps the
310-
captured geometry (`2006x2852`, mean luma 241.42). Without it the touch-overlay exporter loses the
311-
track geometry, and on a long clip the frames too.
312-
313-
The trigger is the overlay drawing touch events, not panel rotation. Measured on an iPhone 17
314-
(iOS 27.0), which has no rotated panel: four seconds with no interaction exports `1206x2622`
315-
intact, ten seconds containing two taps exports `220x480`, and the same two taps under
316-
`--hide-touches` export `1206x2622` with the screen content changing across frames. A 97-second
317-
recording with touches exported `480x220` at mean luma 0.00 throughout. An earlier draft of this
318-
section blamed the inner panel's `rot90` track, which was wrong: every failing sample then available
319-
had merely been captured on that panel, and the one non-rotated sample that looked intact had
320-
contained no touches to draw. Feeding an untouched raw `simctl` capture straight into
321-
`recording-overlay.swift` reproduces a `0x0` zero-duration output. Tracked in #2707.
310+
captured geometry (`2006x2852`, mean luma 241.42). Without it, and before #2707 was fixed, the
311+
touch-overlay exporter lost the track geometry, and on a long clip the frames too.
312+
313+
### The touch-overlay export lost geometry for any capture, not a rotated one (#2707)
314+
315+
The trigger was the overlay drawing touch events, not panel rotation. The #2707 report measured this
316+
on an iPhone 17 simulator (iOS 27.0, non-rot90 panel), Xcode 27.1 beta: four seconds with no
317+
interaction exported `1206x2622` intact, ten seconds containing two taps exported `220x480`, and the
318+
same two taps under `--hide-touches` exported `1206x2622` with the screen content changing across
319+
frames. A 97-second recording with touches exported `480x220` at mean luma 0.00 throughout.
320+
321+
An earlier draft of this section blamed the inner panel's `rot90` track. That was wrong and predated
322+
this matrix: every failing sample then available had merely been captured on that panel, and the one
323+
non-rotated sample that looked intact had contained no touches to draw. Feeding an untouched raw
324+
`simctl` capture straight into `recording-overlay.swift` just copies it through, which is why
325+
`--hide-touches` and an empty gesture list kept the capture intact — the collapse lived in the
326+
overlay export path, not the panel.
327+
328+
The cause was the export preset, measured against a synthetic capture on this host: the burn-in is a
329+
full re-encode through `AVAssetExportSession`, and the default `medium` tier selected
330+
`AVAssetExportPresetMediumQuality`, a fixed-canvas preset that rescales the long edge to 480px — a
331+
`1206x2622` capture lands on `220x480`, a landscape capture on `480x220`. Only
332+
`AVAssetExportPresetHighestQuality` preserves arbitrary capture geometry, and the hardware encoder
333+
makes the full-resolution re-encode cheap: 90s at `1206x2622` re-encoded in ~1–2s, so `high` was
334+
never actually slower.
335+
336+
`recording-overlay.swift` now always exports through the geometry-preserving preset at both quality
337+
tiers, so `--quality` no longer trades capture resolution away. It also verifies its own output
338+
before the caller adopts it — the composited track's resolved size must match the capture, and a
339+
track that went uniformly black while the raw had visible content is rejected — and on either failure
340+
it throws instead of publishing a broken file, so the overlay is dropped, the raw capture is kept,
341+
and the choice is reported on the `record stop` response as `overlayWarning`. Feeding the fixed tool
342+
the same synthetic captures re-measures the failing rows as matching `--hide-touches`:
343+
344+
| Overlay | Interactions | Exported size (before → after) | Black? |
345+
| --- | --- | --- | --- |
346+
| default | two taps, 1206x2622 source | `220x480` → `1206x2622` | no |
347+
| default | taps + scroll, landscape source | `480x220` → source size | no |
348+
| `--hide-touches` | two taps | `1206x2622` → `1206x2622` | no |
349+
350+
The `after` column is this host's offline synthetic harness, which reproduces the collapse and
351+
confirms the fix preserves geometry. The completion condition's on-device re-measure of these rows on
352+
a non-rot90 target and on the Duo inner panel is carried by `test/integration/recording-overlay.test.ts`,
353+
a device-lane case gated behind `AGENT_DEVICE_RECORDING_E2E` that compares a touched export to a
354+
`--hide-touches` control, asserts the same track size, and asserts the overlay actually drew.
355+
356+
The device-lane test reads each export's transform-applied size back through AVFoundation —
357+
`test/integration/support/recording-inspect.swift` reports `renderWidth`/`renderHeight` from the same
358+
`resolvedRenderSize` the overlay tool uses — so a `rot90` panel is compared upright rather than on its
359+
sideways coding grid, and no second MP4 geometry parser is added to the shipped packages.
322360

323361
## Verified on a booted Duo
324362

‎docs/agents/device-verification.md‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -81,12 +81,15 @@ toolchain per command:
8181
three poses, active panel capture, and an app interaction. Duo coverage remains local until GHA
8282
supports the runtime. To inspect the angle independently:
8383
`xcrun devicectl device motion hinge-angle --device <udid> --session-timeout 1 --timeout 5`.
84-
- When a recording must show touches, assume it cannot. The touch-overlay exporter loses the track
85-
geometry whenever it has touch events to draw — `220x480` on a plain iPhone 17 as well as on the
86-
inner panel — and returns all-black frames on long clips, always with exit 0. Record with
87-
`record start --hide-touches` and make the interaction legible through its on-screen effect
88-
(typed text, navigation, a counter) instead of a cursor. The raw `simctl` capture behind it is
89-
correct. See ADR 0025 and #2707.
84+
- Touch overlays export at the captured track size again (#2707). The burn-in used to re-encode
85+
through a fixed 480px preset, so a recording with touches collapsed to `220x480` (landscape
86+
`480x220`) on any panel — not just the `rot90` inner one — and went all-black on long clips,
87+
always with exit 0. That preset is gone: both quality tiers export through the one
88+
geometry-preserving preset, and the compositor now checks its own output (size and non-black)
89+
against the raw before publishing it — on failure it drops the overlay, keeps the raw capture, and
90+
reports it as `overlayWarning` on `record stop` rather than returning a broken file. Only reach
91+
for `record start --hide-touches` when you want the fastest raw capture, not to dodge the defect.
92+
See ADR 0025 and #2707.
9093
- An app must adopt the UIScene lifecycle to launch on iOS 27.1 at all: a legacy
9194
`UIApplicationDelegate` app traps at launch inside
9295
`___UIApplicationEvaluateRuntimeIssueForNoSceneLifecycleAdoption`, which reads like a broken

‎packages/command-registry/src/flag-definitions-action.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ export const ACTION_FLAG_DEFINITIONS: readonly FlagDefinition[] = [
5252
type: 'string',
5353
usageLabel: '--quality <medium|high>',
5454
usageDescription:
55-
'Record: output quality preset; Android maps this to screenrecord bitrate, Apple targets use it for export/encoding, Limrun sessions map medium to recorder quality 5 and high to 8. Legacy numeric values 5-7 map to medium; 8-10 map to high',
55+
'Record: output quality preset; Android maps this to screenrecord bitrate, Limrun sessions map medium to recorder quality 5 and high to 8; Apple export always preserves the captured resolution, so it has no effect there. Legacy numeric values 5-7 map to medium; 8-10 map to high',
5656
projectConfig: true,
5757
recorded: true,
5858
},

0 commit comments

Comments
 (0)