Skip to content

Commit 14e1bd7

Browse files
authored
fix(recording): render the touch overlay at most 30 fps and inside the record request (#3219)
* fix(recording): render the touch overlay at most 30 fps and inside the record request * refactor(recording): take the overlay deadline from host-kit; check its budget against record's envelope - `exportProcessedVideo` uses `Deadline.fromTimeoutMs(...).remainingMs()` instead of a hand-rolled `Date.now() + budgetMs`. - The budget's comment no longer says the client resets the daemon: since #3199 `record` keeps it on timeout, and the caller gets "Daemon request timed out" while the daemon finishes the stop. - `HELPER_EXIT_GRACE_MS` names what it covers: start-up, then verifying the output or cancelling the export, and exiting. - The check that the budget fits `record stop`'s envelope moves from a 90_000 literal in `overlay.test.ts` to the root timeout-policy test, against `record`'s resolved envelope. * test(recording): the overlay budget leaves record stop a 20 s reserve inside its envelope The check let the budget take all but a millisecond of the envelope; the recorder stop, the copies and the playability checks around the overlay need room beyond it.
1 parent 3f3594e commit 14e1bd7

5 files changed

Lines changed: 134 additions & 19 deletions

File tree

‎apple/runner/AgentDeviceRunner/RecordingScripts/RecordingExportSupport.swift‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -88,18 +88,23 @@ func makeRecordingExporter(
8888
return exporter
8989
}
9090

91-
/// Bounded asynchronous export: signals completion through a semaphore and cancels after 120s so a
92-
/// wedged encoder cannot hang the recording pipeline past the caller's own timeout budget.
91+
/// How long an export may run when the caller names no budget of its own.
92+
let defaultRecordingExportTimeoutSeconds: Double = 120
93+
94+
/// Bounded asynchronous export: signals completion through a semaphore and cancels after
95+
/// `timeoutSeconds` so a wedged or slow encoder cannot hang the recording pipeline past the
96+
/// caller's own timeout budget.
9397
func runRecordingExport(
9498
_ exporter: AVAssetExportSession,
99+
timeoutSeconds: Double = defaultRecordingExportTimeoutSeconds,
95100
timeoutMessage: String,
96101
failureMessage: String
97102
) throws {
98103
let semaphore = DispatchSemaphore(value: 0)
99104
exporter.exportAsynchronously {
100105
semaphore.signal()
101106
}
102-
if semaphore.wait(timeout: .now() + 120) == .timedOut {
107+
if semaphore.wait(timeout: .now() + timeoutSeconds) == .timedOut {
103108
exporter.cancelExport()
104109
throw RecordingScriptError.exportFailed(timeoutMessage)
105110
}

‎apple/runner/AgentDeviceRunner/RecordingScripts/recording-overlay.swift‎

Lines changed: 27 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,7 @@ func run() throws {
145145
)
146146
try runRecordingExport(
147147
exporter,
148+
timeoutSeconds: parsedArgs.timeoutSeconds,
148149
timeoutMessage: "Touch overlay export timed out.",
149150
failureMessage: "Touch overlay export failed."
150151
)
@@ -157,10 +158,11 @@ func run() throws {
157158

158159
func parseArguments(
159160
_ arguments: [String]
160-
) throws -> (inputPath: String, outputPath: String, eventsPath: String) {
161+
) throws -> (inputPath: String, outputPath: String, eventsPath: String, timeoutSeconds: Double) {
161162
var inputPath: String?
162163
var outputPath: String?
163164
var eventsPath: String?
165+
var timeoutSeconds = defaultRecordingExportTimeoutSeconds
164166
var index = 0
165167

166168
while index < arguments.count {
@@ -185,17 +187,24 @@ func parseArguments(
185187
throw RecordingScriptError.invalidArgs("--quality must be one of: medium, high")
186188
}
187189
index += 2
190+
case "--timeout-ms":
191+
let rawValue = try recordingOptionValue(arguments, nextIndex, "--timeout-ms")
192+
guard let milliseconds = Double(rawValue), milliseconds > 0 else {
193+
throw RecordingScriptError.invalidArgs("--timeout-ms must be a positive number")
194+
}
195+
timeoutSeconds = milliseconds / 1000
196+
index += 2
188197
default:
189198
throw RecordingScriptError.invalidArgs("Unknown argument: \(argument)")
190199
}
191200
}
192201

193202
guard let inputPath, let outputPath, let eventsPath else {
194203
throw RecordingScriptError.invalidArgs(
195-
"Usage: recording-overlay.swift --input <video> --output <video> --events <json> [--quality <medium|high>]"
204+
"Usage: recording-overlay.swift --input <video> --output <video> --events <json> [--quality <medium|high>] [--timeout-ms <ms>]"
196205
)
197206
}
198-
return (inputPath, outputPath, eventsPath)
207+
return (inputPath, outputPath, eventsPath, timeoutSeconds)
199208
}
200209

201210
/// The composited overlay must keep the captured track's dimensions, so it can only use a preset
@@ -329,19 +338,26 @@ func frameMeanLuma(_ image: CGImage) -> Double? {
329338
return total / Double(count)
330339
}
331340

341+
/// The composited export renders at most this many frames a second, which is smooth for a touch
342+
/// that stays visible for 0.45s.
343+
let maximumCompositedFrameRate: Int32 = 30
344+
345+
/// A variable-frame-rate capture (simctl recordVideo, adb screenrecord) reports one tick of its
346+
/// timescale as `minFrameDuration` (1/600s from simctl). Rendering at that asked the compositor for
347+
/// every frame the encoder could make, 75 a second from an iOS simulator and 60 from an Android
348+
/// emulator, about three times the frames the capture holds; a 5-minute recording's export then
349+
/// outlasted the `record stop` request. So the frame duration is the capture's, but never shorter
350+
/// than one frame at `maximumCompositedFrameRate`. A capture that reports none renders at that
351+
/// rate: its average (`nominalFrameRate`) can be a couple of frames a second over a still screen,
352+
/// which would skip a touch.
332353
func resolvedFrameDuration(for track: AVAssetTrack) -> CMTime {
354+
let shortest = CMTime(value: 1, timescale: maximumCompositedFrameRate)
333355
let minFrameDuration = track.minFrameDuration
334356
if minFrameDuration.isValid && !minFrameDuration.isIndefinite && minFrameDuration.seconds > 0 {
335-
return minFrameDuration
336-
}
337-
338-
let nominalFrameRate = track.nominalFrameRate
339-
if nominalFrameRate > 0 {
340-
let timescale = Int32(max(1, round(nominalFrameRate)))
341-
return CMTime(value: 1, timescale: timescale)
357+
return CMTimeMaximum(minFrameDuration, shortest)
342358
}
343359

344-
return CMTime(value: 1, timescale: 60)
360+
return shortest
345361
}
346362

347363
func overlayPoint(event: GestureEvent, x: Double, y: Double, renderSize: CGSize) -> CGPoint {

‎packages/capture-kit/src/recording/__tests__/overlay.test.ts‎

Lines changed: 45 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ vi.mock('../video.ts', () => ({
2828
waitForPlayableVideo: vi.fn(async () => {}),
2929
}));
3030

31-
import { overlayRecordingTouches } from '../overlay.ts';
31+
import { OVERLAY_BUDGET_MS, overlayRecordingTouches } from '../overlay.ts';
3232
import { AppError } from '@agent-device/kernel/errors';
3333
import { runCmd } from '@agent-device/host-kit/command';
3434

@@ -48,6 +48,7 @@ beforeEach(() => {
4848
});
4949

5050
afterEach(() => {
51+
vi.useRealTimers();
5152
fs.rmSync(tmpDir, { recursive: true, force: true });
5253
});
5354

@@ -129,3 +130,46 @@ test('overlay forwards the requested high export preset', async () => {
129130
expect.arrayContaining(['--events', telemetryPath, '--quality', 'high']),
130131
);
131132
});
133+
134+
test('overlay hands the helper what is left of its budget, inside the record request', async () => {
135+
const videoPath = path.join(tmpDir, 'recording.mp4');
136+
const telemetryPath = path.join(tmpDir, 'recording.gesture-telemetry.json');
137+
fs.writeFileSync(videoPath, 'original');
138+
fs.writeFileSync(telemetryPath, '{"events":[]}');
139+
140+
await overlayRecordingTouches({ videoPath, telemetryPath });
141+
142+
const compileCall = mockRunCmd.mock.calls.find(([cmd]) => cmd === 'xcrun');
143+
const helperCall = mockRunCmd.mock.calls.find(([cmd]) => cmd !== 'xcrun');
144+
const args = helperScriptArgs();
145+
const exportMs = Number(args[args.indexOf('--timeout-ms') + 1]);
146+
expect(exportMs).toBeGreaterThan(0);
147+
// The compile, the export and the helper's exit all fit in the budget. That the budget fits in
148+
// `record`'s request envelope is checked where the envelope is declared.
149+
expect(compileCall?.[2]?.timeoutMs).toBeLessThanOrEqual(OVERLAY_BUDGET_MS);
150+
expect(helperCall?.[2]?.timeoutMs).toBeLessThanOrEqual(OVERLAY_BUDGET_MS);
151+
expect(exportMs).toBeLessThan(helperCall?.[2]?.timeoutMs ?? 0);
152+
});
153+
154+
test('overlay leaves the video as recorded when compiling the helper spent its budget', async () => {
155+
vi.useFakeTimers({ toFake: ['Date'] });
156+
const videoPath = path.join(tmpDir, 'recording.mp4');
157+
const telemetryPath = path.join(tmpDir, 'recording.gesture-telemetry.json');
158+
fs.writeFileSync(videoPath, 'original');
159+
fs.writeFileSync(telemetryPath, '{"events":[]}');
160+
161+
mockRunCmd.mockImplementationOnce(async (_cmd, args) => {
162+
vi.setSystemTime(Date.now() + OVERLAY_BUDGET_MS);
163+
const outputPath = args[args.indexOf('-o') + 1]!;
164+
fs.writeFileSync(outputPath, 'compiled');
165+
fs.chmodSync(outputPath, 0o755);
166+
return { stdout: '', stderr: '', exitCode: 0 };
167+
});
168+
169+
await expect(overlayRecordingTouches({ videoPath, telemetryPath })).rejects.toMatchObject({
170+
code: 'COMMAND_FAILED',
171+
message: 'Failed to add touch overlays to the recording',
172+
});
173+
expect(mockRunCmd.mock.calls.filter(([cmd]) => cmd !== 'xcrun')).toHaveLength(0);
174+
expect(fs.readFileSync(videoPath, 'utf8')).toBe('original');
175+
});

‎packages/capture-kit/src/recording/overlay.ts‎

Lines changed: 41 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import fs from 'node:fs';
22
import path from 'node:path';
33
import { runCmd } from '@agent-device/host-kit/command';
4+
import { Deadline } from '@agent-device/host-kit/retry';
45
import { AppError } from '@agent-device/kernel/errors';
56
import { findProjectRoot } from '@agent-device/host-kit/version';
67
import {
@@ -23,6 +24,20 @@ export function getRecordingOverlaySupportWarning(
2324
return 'touch overlay burn-in is only available on macOS hosts; returning raw video plus gesture telemetry';
2425
}
2526

27+
/**
28+
* `record stop` is one daemon request, which the client abandons after `record`'s 90 s envelope:
29+
* the caller gets "Daemon request timed out" while the daemon finishes the stop. So the overlay must
30+
* end well inside it. Compiling the helper, its export and its exit all come out of this budget;
31+
* past it the stop keeps the raw video and its gesture telemetry, with a warning.
32+
*/
33+
export const OVERLAY_BUDGET_MS = 70_000;
34+
35+
/**
36+
* The helper's time outside its export: starting up, then verifying the composited output or
37+
* cancelling the export, and exiting.
38+
*/
39+
const HELPER_EXIT_GRACE_MS = 5_000;
40+
2641
let overlayScriptPath: string | undefined;
2742
let exportSupportScriptPath: string | undefined;
2843

@@ -44,8 +59,10 @@ async function exportProcessedVideo(params: {
4459
scriptPath: string;
4560
scriptArgs: string[];
4661
commandDescription: string;
62+
budgetMs: number;
4763
}): Promise<void> {
4864
const { videoPath, scriptPath, scriptArgs, commandDescription } = params;
65+
const deadline = Deadline.fromTimeoutMs(params.budgetMs);
4966
await waitForStableFile(videoPath);
5067
await waitForPlayableVideo(videoPath);
5168

@@ -54,11 +71,30 @@ async function exportProcessedVideo(params: {
5471
const executablePath = await compileSwiftSourceFile({
5572
sourcePath: scriptPath,
5673
extraSourcePaths: [getExportSupportScriptPath()],
74+
// `runCmd` reads a timeout of 0 as none.
75+
timeoutMs: Math.max(1, deadline.remainingMs()),
5776
});
58-
await runCmd(executablePath, ['--input', videoPath, '--output', outputPath, ...scriptArgs], {
59-
timeoutMs: 120_000,
60-
env: buildSwiftToolEnv(),
61-
});
77+
const helperMs = deadline.remainingMs();
78+
const exportMs = helperMs - HELPER_EXIT_GRACE_MS;
79+
if (exportMs <= 0) {
80+
throw new AppError(
81+
'COMMAND_FAILED',
82+
`No time was left for the export within the ${params.budgetMs}ms budget`,
83+
);
84+
}
85+
await runCmd(
86+
executablePath,
87+
[
88+
'--input',
89+
videoPath,
90+
'--output',
91+
outputPath,
92+
...scriptArgs,
93+
'--timeout-ms',
94+
String(Math.round(exportMs)),
95+
],
96+
{ timeoutMs: helperMs, env: buildSwiftToolEnv() },
97+
);
6298
await waitForPlayableVideo(outputPath);
6399
fs.renameSync(outputPath, videoPath);
64100
} catch (error) {
@@ -109,5 +145,6 @@ export async function overlayRecordingTouches(params: {
109145
scriptPath: getOverlayScriptPath(),
110146
scriptArgs: ['--events', telemetryPath, '--quality', exportQuality],
111147
commandDescription: `Failed to add touch overlays to the ${targetLabel}`,
148+
budgetMs: OVERLAY_BUDGET_MS,
112149
});
113150
}

‎src/__tests__/command-descriptor-timeout-policy.test.ts‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import {
99
} from '@agent-device/command-registry/registry';
1010
import { INTERACTION_DISPATCH_PATHS } from '@agent-device/contracts/interaction-guarantees';
1111
import { SELECTOR_PIPELINE_POLICIES } from '@agent-device/selectors/selector-pipeline-policy';
12+
import { OVERLAY_BUDGET_MS } from '@agent-device/capture-kit/recording-overlay';
1213
import {
1314
DEFAULT_TIMEOUT_POLICY,
1415
READINESS_BUDGET_MAX_MS,
@@ -334,6 +335,18 @@ test('snapshot uses the standard daemon request timeout with an explicit overrid
334335
);
335336
});
336337

338+
test('the touch overlay budget leaves record stop room inside its request envelope', () => {
339+
// The recorder stop, the copies and the playability checks around the overlay are outside its
340+
// budget (about 1 s for a 10-minute simulator recording), so the envelope keeps a reserve for
341+
// them beyond it.
342+
const reserveMs = 20_000;
343+
const envelopeMs = resolveCommandRequestTimeoutMs(resolveCommandTimeoutPolicy('record'), {
344+
positionals: ['stop'],
345+
flags: {},
346+
});
347+
assert.ok(envelopeMs !== undefined && OVERLAY_BUDGET_MS + reserveMs <= envelopeMs);
348+
});
349+
337350
test('open and prepare startup budgets keep a client-envelope margin over the daemon deadline', () => {
338351
const base = { positionals: [] as string[], flags: {} };
339352

0 commit comments

Comments
 (0)