Skip to content

Commit b50f71a

Browse files
committed
fix(recording): render the touch overlay at most 30 fps and inside the record request
1 parent 2212eac commit b50f71a

4 files changed

Lines changed: 117 additions & 19 deletions

File tree

‎apple/runner/AgentDeviceRunner/RecordingScripts/RecordingExportSupport.swift‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -88,18 +88,23 @@ func makeRecordingExporter(
8888
return exporter
8989
}
9090

91-
/// Bounded asynchronous export: signals completion through a semaphore and cancels after 120s so a
92-
/// wedged encoder cannot hang the recording pipeline past the caller's own timeout budget.
91+
/// How long an export may run when the caller names no budget of its own.
92+
let defaultRecordingExportTimeoutSeconds: Double = 120
93+
94+
/// Bounded asynchronous export: signals completion through a semaphore and cancels after
95+
/// `timeoutSeconds` so a wedged or slow encoder cannot hang the recording pipeline past the
96+
/// caller's own timeout budget.
9397
func runRecordingExport(
9498
_ exporter: AVAssetExportSession,
99+
timeoutSeconds: Double = defaultRecordingExportTimeoutSeconds,
95100
timeoutMessage: String,
96101
failureMessage: String
97102
) throws {
98103
let semaphore = DispatchSemaphore(value: 0)
99104
exporter.exportAsynchronously {
100105
semaphore.signal()
101106
}
102-
if semaphore.wait(timeout: .now() + 120) == .timedOut {
107+
if semaphore.wait(timeout: .now() + timeoutSeconds) == .timedOut {
103108
exporter.cancelExport()
104109
throw RecordingScriptError.exportFailed(timeoutMessage)
105110
}

‎apple/runner/AgentDeviceRunner/RecordingScripts/recording-overlay.swift‎

Lines changed: 27 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,7 @@ func run() throws {
145145
)
146146
try runRecordingExport(
147147
exporter,
148+
timeoutSeconds: parsedArgs.timeoutSeconds,
148149
timeoutMessage: "Touch overlay export timed out.",
149150
failureMessage: "Touch overlay export failed."
150151
)
@@ -157,10 +158,11 @@ func run() throws {
157158

158159
func parseArguments(
159160
_ arguments: [String]
160-
) throws -> (inputPath: String, outputPath: String, eventsPath: String) {
161+
) throws -> (inputPath: String, outputPath: String, eventsPath: String, timeoutSeconds: Double) {
161162
var inputPath: String?
162163
var outputPath: String?
163164
var eventsPath: String?
165+
var timeoutSeconds = defaultRecordingExportTimeoutSeconds
164166
var index = 0
165167

166168
while index < arguments.count {
@@ -185,17 +187,24 @@ func parseArguments(
185187
throw RecordingScriptError.invalidArgs("--quality must be one of: medium, high")
186188
}
187189
index += 2
190+
case "--timeout-ms":
191+
let rawValue = try recordingOptionValue(arguments, nextIndex, "--timeout-ms")
192+
guard let milliseconds = Double(rawValue), milliseconds > 0 else {
193+
throw RecordingScriptError.invalidArgs("--timeout-ms must be a positive number")
194+
}
195+
timeoutSeconds = milliseconds / 1000
196+
index += 2
188197
default:
189198
throw RecordingScriptError.invalidArgs("Unknown argument: \(argument)")
190199
}
191200
}
192201

193202
guard let inputPath, let outputPath, let eventsPath else {
194203
throw RecordingScriptError.invalidArgs(
195-
"Usage: recording-overlay.swift --input <video> --output <video> --events <json> [--quality <medium|high>]"
204+
"Usage: recording-overlay.swift --input <video> --output <video> --events <json> [--quality <medium|high>] [--timeout-ms <ms>]"
196205
)
197206
}
198-
return (inputPath, outputPath, eventsPath)
207+
return (inputPath, outputPath, eventsPath, timeoutSeconds)
199208
}
200209

201210
/// The composited overlay must keep the captured track's dimensions, so it can only use a preset
@@ -329,19 +338,26 @@ func frameMeanLuma(_ image: CGImage) -> Double? {
329338
return total / Double(count)
330339
}
331340

341+
/// The composited export renders at most this many frames a second, which is smooth for a touch
342+
/// that stays visible for 0.45s.
343+
let maximumCompositedFrameRate: Int32 = 30
344+
345+
/// A variable-frame-rate capture (simctl recordVideo, adb screenrecord) reports one tick of its
346+
/// timescale as `minFrameDuration` (1/600s from simctl). Rendering at that asked the compositor for
347+
/// every frame the encoder could make, 75 a second from an iOS simulator and 60 from an Android
348+
/// emulator, about three times the frames the capture holds; a 5-minute recording's export then
349+
/// outlasted the `record stop` request. So the frame duration is the capture's, but never shorter
350+
/// than one frame at `maximumCompositedFrameRate`. A capture that reports none renders at that
351+
/// rate: its average (`nominalFrameRate`) can be a couple of frames a second over a still screen,
352+
/// which would skip a touch.
332353
func resolvedFrameDuration(for track: AVAssetTrack) -> CMTime {
354+
let shortest = CMTime(value: 1, timescale: maximumCompositedFrameRate)
333355
let minFrameDuration = track.minFrameDuration
334356
if minFrameDuration.isValid && !minFrameDuration.isIndefinite && minFrameDuration.seconds > 0 {
335-
return minFrameDuration
336-
}
337-
338-
let nominalFrameRate = track.nominalFrameRate
339-
if nominalFrameRate > 0 {
340-
let timescale = Int32(max(1, round(nominalFrameRate)))
341-
return CMTime(value: 1, timescale: timescale)
357+
return CMTimeMaximum(minFrameDuration, shortest)
342358
}
343359

344-
return CMTime(value: 1, timescale: 60)
360+
return shortest
345361
}
346362

347363
func overlayPoint(event: GestureEvent, x: Double, y: Double, renderSize: CGSize) -> CGPoint {

‎packages/capture-kit/src/recording/__tests__/overlay.test.ts‎

Lines changed: 46 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ vi.mock('../video.ts', () => ({
2828
waitForPlayableVideo: vi.fn(async () => {}),
2929
}));
3030

31-
import { overlayRecordingTouches } from '../overlay.ts';
31+
import { OVERLAY_BUDGET_MS, overlayRecordingTouches } from '../overlay.ts';
3232
import { AppError } from '@agent-device/kernel/errors';
3333
import { runCmd } from '@agent-device/host-kit/command';
3434

@@ -48,6 +48,7 @@ beforeEach(() => {
4848
});
4949

5050
afterEach(() => {
51+
vi.useRealTimers();
5152
fs.rmSync(tmpDir, { recursive: true, force: true });
5253
});
5354

@@ -129,3 +130,47 @@ test('overlay forwards the requested high export preset', async () => {
129130
expect.arrayContaining(['--events', telemetryPath, '--quality', 'high']),
130131
);
131132
});
133+
134+
test('overlay hands the helper what is left of its budget, inside the record request', async () => {
135+
const videoPath = path.join(tmpDir, 'recording.mp4');
136+
const telemetryPath = path.join(tmpDir, 'recording.gesture-telemetry.json');
137+
fs.writeFileSync(videoPath, 'original');
138+
fs.writeFileSync(telemetryPath, '{"events":[]}');
139+
140+
await overlayRecordingTouches({ videoPath, telemetryPath });
141+
142+
const compileCall = mockRunCmd.mock.calls.find(([cmd]) => cmd === 'xcrun');
143+
const helperCall = mockRunCmd.mock.calls.find(([cmd]) => cmd !== 'xcrun');
144+
const args = helperScriptArgs();
145+
const exportMs = Number(args[args.indexOf('--timeout-ms') + 1]);
146+
expect(exportMs).toBeGreaterThan(0);
147+
// The compile, the export and the helper's exit all fit in the budget, which leaves `record`'s
148+
// default 90s request envelope room for the stop around it.
149+
expect(compileCall?.[2]?.timeoutMs).toBeLessThanOrEqual(OVERLAY_BUDGET_MS);
150+
expect(helperCall?.[2]?.timeoutMs).toBeLessThanOrEqual(OVERLAY_BUDGET_MS);
151+
expect(exportMs).toBeLessThan(helperCall?.[2]?.timeoutMs ?? 0);
152+
expect(OVERLAY_BUDGET_MS).toBeLessThan(90_000);
153+
});
154+
155+
test('overlay leaves the video as recorded when compiling the helper spent its budget', async () => {
156+
vi.useFakeTimers({ toFake: ['Date'] });
157+
const videoPath = path.join(tmpDir, 'recording.mp4');
158+
const telemetryPath = path.join(tmpDir, 'recording.gesture-telemetry.json');
159+
fs.writeFileSync(videoPath, 'original');
160+
fs.writeFileSync(telemetryPath, '{"events":[]}');
161+
162+
mockRunCmd.mockImplementationOnce(async (_cmd, args) => {
163+
vi.setSystemTime(Date.now() + OVERLAY_BUDGET_MS);
164+
const outputPath = args[args.indexOf('-o') + 1]!;
165+
fs.writeFileSync(outputPath, 'compiled');
166+
fs.chmodSync(outputPath, 0o755);
167+
return { stdout: '', stderr: '', exitCode: 0 };
168+
});
169+
170+
await expect(overlayRecordingTouches({ videoPath, telemetryPath })).rejects.toMatchObject({
171+
code: 'COMMAND_FAILED',
172+
message: 'Failed to add touch overlays to the recording',
173+
});
174+
expect(mockRunCmd.mock.calls.filter(([cmd]) => cmd !== 'xcrun')).toHaveLength(0);
175+
expect(fs.readFileSync(videoPath, 'utf8')).toBe('original');
176+
});

‎packages/capture-kit/src/recording/overlay.ts‎

Lines changed: 36 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,17 @@ export function getRecordingOverlaySupportWarning(
2323
return 'touch overlay burn-in is only available on macOS hosts; returning raw video plus gesture telemetry';
2424
}
2525

26+
/**
27+
* `record stop` is one daemon request, which the client abandons after 90 s and then resets the
28+
* daemon (`DEFAULT_TIMEOUT_POLICY`), so the overlay must end well inside it. Compiling the helper,
29+
* its export and its exit all come out of this budget; past it the stop keeps the raw video and its
30+
* gesture telemetry, with a warning.
31+
*/
32+
export const OVERLAY_BUDGET_MS = 70_000;
33+
34+
/** How long the helper has to cancel its export and exit once the export's time is spent. */
35+
const HELPER_EXIT_GRACE_MS = 5_000;
36+
2637
let overlayScriptPath: string | undefined;
2738
let exportSupportScriptPath: string | undefined;
2839

@@ -44,8 +55,10 @@ async function exportProcessedVideo(params: {
4455
scriptPath: string;
4556
scriptArgs: string[];
4657
commandDescription: string;
58+
budgetMs: number;
4759
}): Promise<void> {
4860
const { videoPath, scriptPath, scriptArgs, commandDescription } = params;
61+
const deadline = Date.now() + params.budgetMs;
4962
await waitForStableFile(videoPath);
5063
await waitForPlayableVideo(videoPath);
5164

@@ -54,11 +67,29 @@ async function exportProcessedVideo(params: {
5467
const executablePath = await compileSwiftSourceFile({
5568
sourcePath: scriptPath,
5669
extraSourcePaths: [getExportSupportScriptPath()],
70+
timeoutMs: Math.max(1, deadline - Date.now()),
5771
});
58-
await runCmd(executablePath, ['--input', videoPath, '--output', outputPath, ...scriptArgs], {
59-
timeoutMs: 120_000,
60-
env: buildSwiftToolEnv(),
61-
});
72+
const helperMs = deadline - Date.now();
73+
const exportMs = helperMs - HELPER_EXIT_GRACE_MS;
74+
if (exportMs <= 0) {
75+
throw new AppError(
76+
'COMMAND_FAILED',
77+
`No time was left for the export within the ${params.budgetMs}ms budget`,
78+
);
79+
}
80+
await runCmd(
81+
executablePath,
82+
[
83+
'--input',
84+
videoPath,
85+
'--output',
86+
outputPath,
87+
...scriptArgs,
88+
'--timeout-ms',
89+
String(Math.round(exportMs)),
90+
],
91+
{ timeoutMs: helperMs, env: buildSwiftToolEnv() },
92+
);
6293
await waitForPlayableVideo(outputPath);
6394
fs.renameSync(outputPath, videoPath);
6495
} catch (error) {
@@ -109,5 +140,6 @@ export async function overlayRecordingTouches(params: {
109140
scriptPath: getOverlayScriptPath(),
110141
scriptArgs: ['--events', telemetryPath, '--quality', exportQuality],
111142
commandDescription: `Failed to add touch overlays to the ${targetLabel}`,
143+
budgetMs: OVERLAY_BUDGET_MS,
112144
});
113145
}

0 commit comments

Comments
 (0)