Skip to content

Commit e88dd5b

Browse files
committed
feat(recording): decode contact-sheet frames from an exported clip
1 parent 6c8934a commit e88dd5b

8 files changed

Lines changed: 1004 additions & 0 deletions

File tree

‎apple/runner/AgentDeviceRunner/RecordingScripts/RecordingExportSupport.swift‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,14 @@ enum RecordingScriptError: Error, CustomStringConvertible {
99
case invalidArgs(String)
1010
case missingVideoTrack
1111
case exportFailed(String)
12+
case frameWriteFailed(String)
1213

1314
var description: String {
1415
switch self {
1516
case .invalidArgs(let message):
1617
return message
18+
case .frameWriteFailed(let message):
19+
return message
1720
case .missingVideoTrack:
1821
return "Input video does not contain a video track."
1922
case .exportFailed(let message):
Lines changed: 203 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,203 @@
1+
import AVFoundation
2+
import Foundation
3+
import ImageIO
4+
import UniformTypeIdentifiers
5+
6+
/// Extracts the frames at the requested presentation times from a recording and writes each one as
7+
/// a PNG, then prints a JSON manifest naming the time each returned image actually carries.
8+
///
9+
/// The caller owns the sampling grid: it passes an explicit, bounded list of times, so this script
10+
/// never walks a whole clip. `--times` is the whole job description, which keeps the amount of
11+
/// decoding, the bytes on disk, and the wall clock all bounded by the request rather than by the
12+
/// recording's length.
13+
///
14+
/// Reported times are the generator's *actual* return times, not the ones that were asked for. A
15+
/// decoder answers a request with the frame it holds at that moment; the manifest says which frame
16+
/// that was, so nothing downstream can print a requested time as though it had been decoded.
17+
18+
struct FrameManifest: Encodable {
19+
struct Frame: Encodable {
20+
let index: Int
21+
let requestedTimeMs: Double
22+
let actualTimeMs: Double
23+
let width: Int
24+
let height: Int
25+
let path: String
26+
}
27+
28+
struct Skipped: Encodable {
29+
let index: Int
30+
let requestedTimeMs: Double
31+
let reason: String
32+
}
33+
34+
let inputPath: String
35+
let durationMs: Double
36+
let frames: [Frame]
37+
let skipped: [Skipped]
38+
}
39+
40+
/// Entry point: `@main` because multi-file swiftc compilation reserves top-level statements for
41+
/// `main.swift`, which cannot be shared per-script.
42+
@main
43+
enum RecordingFrames {
44+
static func main() {
45+
do {
46+
try run()
47+
} catch {
48+
fputs("recording-frames: \(error)\n", stderr)
49+
exit(1)
50+
}
51+
}
52+
}
53+
54+
func run() throws {
55+
let arguments = Array(CommandLine.arguments.dropFirst())
56+
let inputURL = URL(fileURLWithPath: try requiredOption(arguments, "--input"))
57+
let outputDirectoryURL = URL(fileURLWithPath: try requiredOption(arguments, "--output-dir"), isDirectory: true)
58+
let requestedTimesMs = try parseTimes(try requiredOption(arguments, "--times"))
59+
let maxWidth = try parseMaxWidth(optionValue(arguments, "--max-width"))
60+
61+
guard !requestedTimesMs.isEmpty else {
62+
throw RecordingScriptError.invalidArgs("--times must name at least one presentation time")
63+
}
64+
try FileManager.default.createDirectory(at: outputDirectoryURL, withIntermediateDirectories: true)
65+
66+
let asset = AVURLAsset(url: inputURL)
67+
_ = try sourceVideoTrack(of: asset)
68+
let durationMs = max(0, asset.duration.seconds * 1000)
69+
70+
let generator = AVAssetImageGenerator(asset: asset)
71+
generator.appliesPreferredTrackTransform = true
72+
generator.maximumSize = CGSize(width: maxWidth, height: maxWidth)
73+
generator.requestedTimeToleranceBefore = .zero
74+
generator.requestedTimeToleranceAfter = .zero
75+
76+
var frames: [FrameManifest.Frame] = []
77+
var skipped: [FrameManifest.Skipped] = []
78+
79+
for (index, requestedTimeMs) in requestedTimesMs.enumerated() {
80+
let requestedTime = CMTime(seconds: requestedTimeMs / 1000, preferredTimescale: 600)
81+
var actualTime = CMTime.invalid
82+
let cgImage: CGImage
83+
do {
84+
cgImage = try generator.copyCGImage(at: requestedTime, actualTime: &actualTime)
85+
} catch {
86+
skipped.append(
87+
FrameManifest.Skipped(
88+
index: index,
89+
requestedTimeMs: requestedTimeMs,
90+
reason: "\(error)"
91+
)
92+
)
93+
continue
94+
}
95+
96+
// A frame the decoder cannot place on the timeline is not a frame at 0 ms. Writing it would
97+
// label a mid-clip moment as the opening one, so the sample goes unanswered instead and the
98+
// manifest says so.
99+
guard CMTIME_IS_VALID(actualTime), actualTime.timescale != 0 else {
100+
skipped.append(
101+
FrameManifest.Skipped(
102+
index: index,
103+
requestedTimeMs: requestedTimeMs,
104+
reason: "Decoder returned no presentation time for this frame"
105+
)
106+
)
107+
continue
108+
}
109+
110+
let outputURL = outputDirectoryURL.appendingPathComponent(frameFileName(index: index))
111+
do {
112+
try writePNG(cgImage: cgImage, to: outputURL)
113+
} catch {
114+
// A frame the decoder produced but storage refused is a failed run, not a moment the
115+
// recording has nothing to show: reporting it as skipped would let a partial sheet claim
116+
// to cover the clip.
117+
try? FileManager.default.removeItem(at: outputURL)
118+
throw RecordingScriptError.frameWriteFailed(
119+
"Could not write frame \(index) at \(requestedTimeMs)ms to \(outputURL.path): \(error)"
120+
)
121+
}
122+
123+
frames.append(
124+
FrameManifest.Frame(
125+
index: index,
126+
requestedTimeMs: requestedTimeMs,
127+
actualTimeMs: timeIntervalMs(actualTime),
128+
width: cgImage.width,
129+
height: cgImage.height,
130+
path: outputURL.path
131+
)
132+
)
133+
}
134+
135+
let manifest = FrameManifest(
136+
inputPath: inputURL.path,
137+
durationMs: durationMs,
138+
frames: frames,
139+
skipped: skipped
140+
)
141+
let encoder = JSONEncoder()
142+
encoder.outputFormatting = [.sortedKeys]
143+
FileHandle.standardOutput.write(try encoder.encode(manifest))
144+
}
145+
146+
func requiredOption(_ arguments: [String], _ flag: String) throws -> String {
147+
guard let index = arguments.firstIndex(of: flag) else {
148+
throw RecordingScriptError.invalidArgs("Missing \(flag)")
149+
}
150+
return try recordingOptionValue(arguments, index + 1, flag)
151+
}
152+
153+
func optionValue(_ arguments: [String], _ flag: String) -> String? {
154+
guard let index = arguments.firstIndex(of: flag), index + 1 < arguments.count else {
155+
return nil
156+
}
157+
return arguments[index + 1]
158+
}
159+
160+
func parseTimes(_ value: String) throws -> [Double] {
161+
let parts = value.split(separator: ",", omittingEmptySubsequences: true)
162+
guard !parts.isEmpty else {
163+
throw RecordingScriptError.invalidArgs("--times must be a comma-separated list of milliseconds")
164+
}
165+
return try parts.map { part in
166+
guard let parsed = Double(part), parsed.isFinite, parsed >= 0 else {
167+
throw RecordingScriptError.invalidArgs("Invalid sample time: \(part)")
168+
}
169+
return parsed
170+
}
171+
}
172+
173+
func parseMaxWidth(_ value: String?) throws -> Int {
174+
guard let value else { return 360 }
175+
guard let parsed = Int(value), parsed > 0 else {
176+
throw RecordingScriptError.invalidArgs("Invalid --max-width: \(value)")
177+
}
178+
return parsed
179+
}
180+
181+
func timeIntervalMs(_ time: CMTime) -> Double {
182+
guard CMTIME_IS_VALID(time), time.timescale != 0 else { return 0 }
183+
return CMTimeGetSeconds(time) * 1000
184+
}
185+
186+
func frameFileName(index: Int) -> String {
187+
String(format: "frame-%04d.png", index)
188+
}
189+
190+
func writePNG(cgImage: CGImage, to url: URL) throws {
191+
guard let destination = CGImageDestinationCreateWithURL(
192+
url as CFURL,
193+
UTType.png.identifier as CFString,
194+
1,
195+
nil
196+
) else {
197+
throw RecordingScriptError.exportFailed("Failed to open PNG destination for \(url.lastPathComponent)")
198+
}
199+
CGImageDestinationAddImage(destination, cgImage, nil)
200+
guard CGImageDestinationFinalize(destination) else {
201+
throw RecordingScriptError.exportFailed("Failed to encode PNG for \(url.lastPathComponent)")
202+
}
203+
}
Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
import { PNG } from './png.ts';
2+
3+
/**
4+
* Decoded-frame builders for tests that reason about pixels: a solid fill is enough to name what a
5+
* test claims a frame contains without shipping a PNG fixture through the repo.
6+
*/
7+
8+
export type Rgba = readonly [number, number, number, number];
9+
10+
export const BLACK: Rgba = [0, 0, 0, 255];
11+
12+
export function solidPng(width: number, height: number, color: Rgba = BLACK): PNG {
13+
const png = new PNG({ width, height });
14+
for (let offset = 0; offset < png.data.length; offset += 4) {
15+
png.data[offset] = color[0];
16+
png.data[offset + 1] = color[1];
17+
png.data[offset + 2] = color[2];
18+
png.data[offset + 3] = color[3];
19+
}
20+
return png;
21+
}

‎packages/capture-kit/src/recording/__tests__/recording-scripts.test.ts‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,9 @@ import assert from 'node:assert/strict';
33
import path from 'node:path';
44
import { fileURLToPath } from 'node:url';
55
import { runCmd } from '@agent-device/host-kit/command';
6+
import { AppError } from '@agent-device/kernel/errors';
7+
import { CONTACT_SHEET_UNSUPPORTED_HOST_REASON } from '../contact-sheet-report.ts';
8+
import { assertContactSheetHostSupport } from '../contact-sheet-frames.ts';
69
import { getRecordingOverlaySupportWarning } from '../overlay.ts';
710

811
const __dirname = path.dirname(fileURLToPath(import.meta.url));
@@ -71,10 +74,38 @@ test(
7174
SWIFT_TYPECHECK_TIMEOUT_MS,
7275
);
7376

77+
test(
78+
'recording frames Swift script typechecks',
79+
async (t) => {
80+
if (process.platform !== 'darwin') {
81+
t.skip('Swift recording scripts are only validated on macOS');
82+
}
83+
84+
await assertSwiftScriptTypechecks(path.join(recordingScriptsDir, 'recording-frames.swift'), [
85+
path.join(recordingScriptsDir, 'RecordingExportSupport.swift'),
86+
]);
87+
},
88+
SWIFT_TYPECHECK_TIMEOUT_MS,
89+
);
90+
7491
test('recording overlays are explicitly unsupported on non-macOS hosts', () => {
7592
assert.equal(
7693
getRecordingOverlaySupportWarning('linux'),
7794
'touch overlay burn-in is only available on macOS hosts; returning raw video plus gesture telemetry',
7895
);
7996
assert.equal(getRecordingOverlaySupportWarning('darwin'), undefined);
8097
});
98+
99+
test('contact sheets are explicitly unsupported on non-macOS hosts', () => {
100+
assert.throws(
101+
() => assertContactSheetHostSupport('linux'),
102+
(error: unknown) => {
103+
return (
104+
error instanceof AppError &&
105+
error.code === 'UNSUPPORTED_OPERATION' &&
106+
error.details?.reason === CONTACT_SHEET_UNSUPPORTED_HOST_REASON
107+
);
108+
},
109+
);
110+
assert.doesNotThrow(() => assertContactSheetHostSupport('darwin'));
111+
});

0 commit comments

Comments
 (0)