Skip to content

Commit 14e598e

Browse files
feat(media-use): your own voice recording is a voice line, and speed applies to it (#7377)
* feat(media-use): your own voice recording is a voice line, and speed applies to it * fix(media-use): regenerate the skills manifest and keep audio.mjs's header at its limit
1 parent b418bb2 commit 14e598e

7 files changed

Lines changed: 266 additions & 31 deletions

File tree

‎skills-manifest.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,8 @@
1818
"files": 13
1919
},
2020
"media-use": {
21-
"hash": "44ef2d115fa8bebd",
22-
"files": 118
21+
"hash": "d4221a4cd4118c29",
22+
"files": 120
2323
},
2424
"remotion-to-hyperframes": {
2525
"hash": "73b82d19db2ed8c9",

‎skills/media-use/audio/scripts/audio.mjs‎

Lines changed: 53 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ import {
5252
} from "./lib/tts.mjs";
5353
import { generateBgmDetached, inferBgmPrompt, retrieveBgm } from "./lib/bgm.mjs";
5454
import { BUNDLED_SFX_DIR, resolveSfx } from "./lib/sfx.mjs";
55+
import { parseWords, scaleWords, stretchToWav } from "./lib/recording.mjs";
5556
import { mapWithConcurrency } from "./lib/concurrency.mjs";
5657
import { openAudioMeta } from "./lib/audio-meta.mjs";
5758
import { recordInManifest, voicePaths, writtenAssets } from "./lib/media-record.mjs";
@@ -127,42 +128,63 @@ let voices = prev.voices ?? [];
127128
let ttsProvider = prev.tts_provider ?? null;
128129
let voiceId = prev.voice_id ?? null;
129130
if (only.has("tts") && lines.length) {
130-
try {
131-
ttsProvider = pickProvider(
132-
providerOverride || (request.provider === "auto" ? null : request.provider),
133-
);
134-
} catch (e) {
135-
die(e.message);
131+
if (lines.some((l) => !l.audio)) {
132+
try {
133+
ttsProvider = pickProvider(
134+
providerOverride || (request.provider === "auto" ? null : request.provider),
135+
);
136+
} catch (e) {
137+
die(e.message);
138+
}
139+
voiceId = await resolveVoiceId({
140+
provider: ttsProvider,
141+
userVoice: voiceOverride || request.voice,
142+
lang,
143+
});
144+
} else {
145+
ttsProvider = "recording"; // every line is the person's own recording: no TTS provider to pick
146+
voiceId = null;
136147
}
137-
voiceId = await resolveVoiceId({
138-
provider: ttsProvider,
139-
userVoice: voiceOverride || request.voice,
140-
lang,
141-
});
142148
console.error(`· tts: ${ttsProvider} · voice ${voiceId} · ${lines.length} line(s)`);
143149
const paths = voicePaths(hyperframesDir, lines, anomalies);
150+
// A line with `audio` is the person's own recording (path relative to the project), with optional
151+
// `words`; `speed` applies to it like to a generated voice. Returns synthesizeOne's shape.
152+
const useRecording = (line, abs) => {
153+
const made = stretchToWav(resolve(hyperframesDir, String(line.audio)), abs, speed);
154+
if (!made.ok) return made;
155+
try {
156+
const given = parseWords(line.words);
157+
return { ok: true, words: given && scaleWords(given, speed), source: "recording" };
158+
} catch (e) {
159+
return { ok: false, error: e.message };
160+
}
161+
};
144162
const synthLine = async (line) => {
145163
const id = String(line.id);
146164
const text = String(line.text ?? "").trim();
147-
if (!text) {
165+
if (!text && !line.audio) {
148166
anomalies.push(`line ${id}: empty text — skipped`);
149167
return null;
150168
}
151169
const rel = paths.get(id);
152170
const abs = join(hyperframesDir, rel);
153-
const { ok, words, error } = await synthesizeOne({
154-
provider: ttsProvider,
155-
text,
156-
voiceId,
157-
lang,
158-
speed,
159-
model: flag("tts-model", request.tts_model),
160-
style: flag("style", line.style ?? request.style),
161-
wavAbs: abs,
162-
hyperframesDir,
163-
});
171+
const { ok, words, error, source } = line.audio
172+
? useRecording(line, abs)
173+
: await synthesizeOne({
174+
provider: ttsProvider,
175+
text,
176+
voiceId,
177+
lang,
178+
speed,
179+
model: flag("tts-model", request.tts_model),
180+
style: flag("style", line.style ?? request.style),
181+
wavAbs: abs,
182+
hyperframesDir,
183+
});
164184
if (!ok) {
165-
anomalies.push(`line ${id}: TTS failed — omitted${error ? ` (${error})` : ""}`);
185+
anomalies.push(
186+
`line ${id}: ${line.audio ? "recording" : "TTS"} failed — omitted${error ? ` (${error})` : ""}`,
187+
);
166188
return null;
167189
}
168190
let wordArr = words; // heygen: native; else transcribe
@@ -172,7 +194,13 @@ if (only.has("tts") && lines.length) {
172194
anomalies.push(`line ${id}: bad voice duration — omitted`);
173195
return null;
174196
}
175-
return { id, path: rel, duration_s: r3(dur), words: withWordIds(wordArr) };
197+
return {
198+
id,
199+
path: rel,
200+
duration_s: r3(dur),
201+
words: withWordIds(wordArr),
202+
...(source && { source }),
203+
};
176204
};
177205
const results = await mapWithConcurrency(lines, ttsConcurrency, synthLine);
178206
voices = results.filter(Boolean);

‎skills/media-use/audio/scripts/lib/media-record.mjs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ export function agentWritePath(
3535
export function voicePaths(hyperframesDir, lines, anomalies) {
3636
const taken = new Set();
3737
const paths = new Map();
38-
for (const line of lines.filter((l) => String(l.text ?? "").trim())) {
38+
for (const line of lines.filter((l) => String(l.text ?? "").trim() || l.audio)) {
3939
const rel = agentWritePath(hyperframesDir, `assets/voice/${line.id}.wav`, { anomalies, taken });
4040
taken.add(rel);
4141
paths.set(String(line.id), rel);
@@ -60,9 +60,9 @@ export function writtenAssets({ only, lines, voices, ttsProvider, bgm, bgmFields
6060
path: voice.path,
6161
type: "voice",
6262
source: "generated",
63-
intent: textById.get(voice.id),
63+
intent: textById.get(voice.id) || "your own recording",
6464
duration: voice.duration_s,
65-
provider: ttsProvider,
65+
provider: voice.source ?? ttsProvider,
6666
});
6767
}
6868
}

‎skills/media-use/audio/scripts/lib/media-record.test.mjs‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ import { test } from "node:test";
33
import { mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
44
import { join } from "node:path";
55
import { tmpdir } from "node:os";
6-
import { findByPrompt, readManifest } from "../../../scripts/lib/manifest.mjs";
6+
import { AGENT_SOURCES, findByPrompt, readManifest } from "../../../scripts/lib/manifest.mjs";
77
import { agentWritePath, recordInManifest, voicePaths, writtenAssets } from "./media-record.mjs";
88

99
const noBgm = { bgm: null, bgmFields: { bgm_pending: false } };
@@ -222,3 +222,18 @@ test("a record never takes the id of a download still in flight", (t) => {
222222
["bgm_002"],
223223
);
224224
});
225+
226+
// An engine-written copy stays an agent source, or every rerun would write a numbered copy beside it
227+
// (AGENT_SOURCES, manifest.mjs); what says it is the person's own voice is the provider and intent.
228+
test("a recording line is recorded as your own recording, still an engine-written file", () => {
229+
const [asset] = writtenAssets({
230+
only: new Set(["tts"]),
231+
lines: [{ id: "01", audio: "take.m4a" }],
232+
voices: [{ id: "01", path: "assets/voice/01.wav", duration_s: 2, source: "recording" }],
233+
ttsProvider: "recording",
234+
});
235+
236+
assert.equal(asset.provider, "recording");
237+
assert.equal(asset.intent, "your own recording");
238+
assert.ok(AGENT_SOURCES.includes(asset.source));
239+
});
Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
// recording.mjs — the person's own voice recording as a voice line: converted to the engine's wav,
2+
// time-stretched by `speed` with the pitch kept, with word timings scaled to match.
3+
import { spawnSync } from "node:child_process";
4+
import { existsSync, mkdirSync } from "node:fs";
5+
import { dirname } from "node:path";
6+
7+
// ffmpeg's atempo takes 0.5..2 per stage; chain stages for the rest of the 0.1..3 speed range.
8+
export function atempoChain(speed) {
9+
const stages = [];
10+
let left = speed;
11+
while (left > 2) {
12+
stages.push(2);
13+
left /= 2;
14+
}
15+
while (left < 0.5) {
16+
stages.push(0.5);
17+
left /= 0.5;
18+
}
19+
if (Math.abs(left - 1) > 1e-9 || stages.length === 0) stages.push(left);
20+
return stages.map((s) => `atempo=${Number(s.toFixed(6))}`).join(",");
21+
}
22+
23+
/** src -> 44.1k mono wav at wavAbs, played `speed` times faster. Returns { ok, error? }. */
24+
export function stretchToWav(src, wavAbs, speed = 1) {
25+
if (!existsSync(src)) return { ok: false, error: `recording not found: ${src}` };
26+
mkdirSync(dirname(wavAbs), { recursive: true });
27+
const filter = speed === 1 ? [] : ["-af", atempoChain(speed)];
28+
const r = spawnSync(
29+
"ffmpeg",
30+
["-y", "-loglevel", "error", "-i", src, ...filter, "-ar", "44100", "-ac", "1", wavAbs],
31+
{ encoding: "utf8" },
32+
);
33+
if (r.error || r.status !== 0 || !existsSync(wavAbs)) {
34+
return { ok: false, error: `ffmpeg could not convert ${src}: ${r.error?.message ?? r.stderr}` };
35+
}
36+
return { ok: true };
37+
}
38+
39+
/** Caller-supplied word timings ({text|word, start, end}) as [{text,start,end}], or null when absent. */
40+
export function parseWords(words) {
41+
if (!Array.isArray(words) || words.length === 0) return null;
42+
return words.map((w, i) => {
43+
const text = w?.text ?? w?.word;
44+
if (typeof text !== "string" || !Number.isFinite(w.start) || !Number.isFinite(w.end)) {
45+
throw new Error(`words[${i}] needs text (or word), start and end in seconds`);
46+
}
47+
return { text, start: w.start, end: w.end };
48+
});
49+
}
50+
51+
/** Words of a recording played `speed` times faster land at start/speed. */
52+
export const scaleWords = (words, speed) =>
53+
words.map((w) => ({ ...w, start: w.start / speed, end: w.end / speed }));
Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
import { test } from "node:test";
2+
import assert from "node:assert/strict";
3+
import { spawnSync } from "node:child_process";
4+
import { mkdtempSync, readFileSync, writeFileSync, rmSync } from "node:fs";
5+
import { join, dirname } from "node:path";
6+
import { tmpdir } from "node:os";
7+
import { fileURLToPath } from "node:url";
8+
import { atempoChain, parseWords, scaleWords } from "./lib/recording.mjs";
9+
10+
const AUDIO = join(dirname(fileURLToPath(import.meta.url)), "audio.mjs");
11+
12+
test("atempoChain splits speeds outside ffmpeg's 0.5..2 stage range", () => {
13+
assert.equal(atempoChain(1.5), "atempo=1.5");
14+
assert.equal(atempoChain(3), "atempo=2,atempo=1.5");
15+
assert.equal(atempoChain(0.25), "atempo=0.5,atempo=0.5");
16+
});
17+
18+
test("parseWords takes text or word and rejects a missing time", () => {
19+
assert.deepEqual(parseWords([{ word: "hi", start: 0, end: 1 }]), [
20+
{ text: "hi", start: 0, end: 1 },
21+
]);
22+
assert.equal(parseWords(undefined), null);
23+
assert.throws(() => parseWords([{ text: "hi", start: 0 }]), /words\[0\]/);
24+
});
25+
26+
test("scaleWords moves words by 1/speed", () => {
27+
assert.deepEqual(scaleWords([{ text: "a", start: 1, end: 2 }], 2), [
28+
{ text: "a", start: 0.5, end: 1 },
29+
]);
30+
});
31+
32+
test("a recording line with malformed words is omitted with the reason, not a crash", () => {
33+
const dir = mkdtempSync(join(tmpdir(), "hf-rec-"));
34+
try {
35+
const make = spawnSync(
36+
"ffmpeg",
37+
["-y", "-loglevel", "error", "-f", "lavfi", "-i", "sine=duration=1", join(dir, "take.wav")],
38+
{ encoding: "utf8" },
39+
);
40+
assert.equal(make.status, 0, make.stderr);
41+
writeFileSync(
42+
join(dir, "audio_request.json"),
43+
JSON.stringify({
44+
lines: [{ id: "01", audio: "take.wav", words: [{ text: "x", start: 0 }] }],
45+
}),
46+
);
47+
const run = spawnSync(
48+
process.execPath,
49+
[
50+
AUDIO,
51+
"--request",
52+
join(dir, "audio_request.json"),
53+
"--hyperframes",
54+
dir,
55+
"--out",
56+
join(dir, "m.json"),
57+
"--only",
58+
"tts",
59+
],
60+
{ encoding: "utf8", env: { ...process.env, HOME: dir } },
61+
);
62+
assert.equal(run.status, 0, run.stderr);
63+
assert.match(run.stdout, /line 01: recording failed — omitted \(words\[0\] needs text/);
64+
assert.deepEqual(JSON.parse(readFileSync(join(dir, "m.json"), "utf8")).voices, []);
65+
} finally {
66+
rmSync(dir, { recursive: true, force: true });
67+
}
68+
});
69+
70+
// A 1 s gap then a 1 s tone is a "word" at 1..2 s. At speed 2 the tone must start at 0.5 s in the
71+
// stretched file, which is where the scaled word says it is.
72+
test("a recording at speed 2: the file is stretched and the words still land on the audio", () => {
73+
const dir = mkdtempSync(join(tmpdir(), "hf-rec-"));
74+
try {
75+
const make = spawnSync(
76+
"ffmpeg",
77+
[
78+
"-y",
79+
"-loglevel",
80+
"error",
81+
"-f",
82+
"lavfi",
83+
"-i",
84+
"anullsrc=r=44100:cl=mono",
85+
"-f",
86+
"lavfi",
87+
"-i",
88+
"sine=frequency=440:duration=1:sample_rate=44100",
89+
"-filter_complex",
90+
"[0]atrim=duration=1[s];[s][1]concat=n=2:v=0:a=1",
91+
join(dir, "take.wav"),
92+
],
93+
{ encoding: "utf8" },
94+
);
95+
assert.equal(make.status, 0, make.stderr);
96+
writeFileSync(
97+
join(dir, "audio_request.json"),
98+
JSON.stringify({
99+
speed: 2,
100+
lines: [{ id: "01", audio: "take.wav", words: [{ text: "tone", start: 1, end: 2 }] }],
101+
}),
102+
);
103+
const run = spawnSync(
104+
process.execPath,
105+
[
106+
AUDIO,
107+
"--request",
108+
join(dir, "audio_request.json"),
109+
"--hyperframes",
110+
dir,
111+
"--out",
112+
join(dir, "audio_meta.json"),
113+
"--only",
114+
"tts",
115+
],
116+
{ encoding: "utf8", env: { ...process.env, HOME: dir } },
117+
);
118+
assert.equal(run.status, 0, run.stderr + run.stdout);
119+
const meta = JSON.parse(readFileSync(join(dir, "audio_meta.json"), "utf8"));
120+
const voice = meta.voices[0];
121+
assert.equal(meta.tts_provider, "recording");
122+
assert.equal(voice.source, "recording");
123+
assert.ok(Math.abs(voice.duration_s - 1) < 0.05, `duration ${voice.duration_s}`);
124+
assert.deepEqual([voice.words[0].start, voice.words[0].end], [0.5, 1]);
125+
const probe = spawnSync(
126+
"ffmpeg",
127+
["-i", join(dir, voice.path), "-af", "silencedetect=noise=-40dB:d=0.2", "-f", "null", "-"],
128+
{ encoding: "utf8" },
129+
);
130+
const silenceEnd = Number(/silence_end: ([\d.]+)/.exec(probe.stderr)?.[1]);
131+
assert.ok(
132+
Math.abs(silenceEnd - voice.words[0].start) < 0.08,
133+
`tone starts at ${silenceEnd}s, word says ${voice.words[0].start}s`,
134+
);
135+
} finally {
136+
rmSync(dir, { recursive: true, force: true });
137+
}
138+
});

‎skills/media-use/references/audio.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ node <SKILL_DIR>/audio/scripts/audio.mjs --request ./audio_request.json --out ./
1010
```
1111

1212
- **Request** `{ provider?, voice?, lang?, speed?, tts_model?, style?, lines: [{ id, text, style?, sfx?: [names] }], bgm: { mode?, query?, prompt? } }`: `id` joins each line back to your model; `bgm.mode` = `retrieve | generate | none` (omit for auto). `--only tts,bgm,sfx` runs a subset and merges into an existing `--out`.
13+
- **Your own recording**: a line with `audio` (a path in the project) instead of `text` is used as the voice: `{ id, audio: "take.m4a", words?: [{ text, start, end }] }`. It is converted to `assets/voice/<id>.wav`, `speed` stretches it with the pitch kept and scales `words` the same (without `words` the recording is transcribed). Its `audio_meta.json` voice has `source: "recording"`; with only recording lines `tts_provider` is `"recording"` and no provider or HeyGen sign-in is needed.
1314
- **Gemini narration** is opt-in with `provider: "gemini"` and an API key or service-account credentials. `tts_model` defaults to `gemini-3.8-flash-tts`; 3.8 Flash-Lite, 3.1 Flash Preview, and 2.5 Pro/Flash Preview TTS are also supported. `style` directs delivery; a line's style overrides the request's style. CLI overrides: `--tts-model`, `--style`. See `audio/references/tts.md` for a complete request. The existing automatic provider order is unchanged.
1415
- **Output** `audio_meta.json` (id-keyed): `voices[].{path,duration_s,words[]}` (word timestamps for captions), `sfx[]`, `bgm`, `total_duration_s`.
1516
- **HeyGen free-usage path**: HeyGen CLI auth unlocks TTS plus music retrieval; SFX come from Openverse with no account (`audio/references/sfx.md`). Local/provider-specific generators are explicit alternatives where installed; run `npx hyperframes media-use resolve --doctor` before assuming retrieval or TTS will work.

0 commit comments

Comments
 (0)