Skip to content

Commit 5420dc8

Browse files
committed
fix(cli): hear a reversed stereo channel, and warn when its peak was skipped
Averaging the channels cancelled a reversed one, so a real hit was reported as silence. The skip warning also compared against the decode margin, so a peak just before the kept window produced no warning.
1 parent d08d972 commit 5420dc8

4 files changed

Lines changed: 91 additions & 21 deletions

File tree

‎packages/cli/src/audio/analysis/run.test.ts‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,24 @@ describe.skipIf(!ffmpeg)("analyzeTargets", () => {
9191
await expect(previewTarget(path, "late-cut.wav", 2, 1)).rejects.toThrow(/leaves nothing/);
9292
});
9393

94+
it("warns when the loudest hit is only inside the margin before the kept window", async () => {
95+
const tone = (at: number, gain: number) => (t: number) =>
96+
t < at || t >= at + 0.03 ? 0 : gain * Math.sin(2 * Math.PI * 1000 * t);
97+
const path = wav("margin.wav", 2.4, (t) => tone(0.502, 0.8)(t) + tone(1.202, 0.25)(t));
98+
const warned = async (skip: number) =>
99+
(
100+
await analyzeTargets([await previewTarget(path, "margin.wav", 4, skip)], {
101+
capabilities: ["moments"],
102+
})
103+
).items[0]!.warnings;
104+
expect(await warned(0.55)).toEqual([
105+
"the loudest part of margin.wav is in the first 0.55 s, which is skipped",
106+
]);
107+
expect(await warned(0.65)).toEqual([
108+
"the loudest part of margin.wav is in the first 0.65 s, which is skipped",
109+
]);
110+
});
111+
94112
it("finds no hit where a window cuts into a sound that is already playing", async () => {
95113
const chord = (t: number) =>
96114
Array.from({ length: 40 }, (_, k) => Math.sin(2 * Math.PI * 110 * (k + 1) * t + k)).reduce(

‎packages/cli/src/audio/analysis/run.ts‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,15 @@ function decodeSpan(target: AudioTarget, stream: AudioStreamInfo) {
105105
const loudest = (signal: AudioSignal) =>
106106
Math.max(0, ...envelope(signal, { hopMs: 10, scale: "linear" }).values);
107107

108+
/** The decoded span without the margin added so an onset at the window edge is not the cut. */
109+
function retained(signal: AudioSignal, before: number, after: number): AudioSignal {
110+
const skip = Math.round(before * signal.sampleRate);
111+
const length = signal.channels[0]?.length ?? 0;
112+
const end = Math.max(skip, length - Math.round(after * signal.sampleRate));
113+
if (!skip && end >= length) return signal;
114+
return { ...signal, channels: signal.channels.map((channel) => channel.subarray(skip, end)) };
115+
}
116+
108117
/** A file placed after cutting its head says so when the cut part held its loudest moment. */
109118
async function skippedPeak(target: AudioTarget, kept: AudioSignal, stream: AudioStreamInfo) {
110119
const skipped = target.kind === "file" ? (target.window?.from ?? 0) : 0;
@@ -139,7 +148,14 @@ async function analyzeTarget(
139148
...analysis.values,
140149
provenance: analysis.provenance,
141150
...(full ? { ladder: analysis.ladder } : {}),
142-
warnings: [...analysis.warnings, ...(await skippedPeak(target, decoded.signal, stream))],
151+
warnings: [
152+
...analysis.warnings,
153+
...(await skippedPeak(
154+
target,
155+
retained(decoded.signal, context.before, context.after),
156+
stream,
157+
)),
158+
],
143159
};
144160
}
145161

‎packages/core/src/audioAnalysis.test.ts‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,21 @@ describe("moments", () => {
7070
});
7171
});
7272

73+
describe("opposite polarity", () => {
74+
const reversed = signal(3, impact(1, 0.01, 0.05));
75+
reversed.channels.push(Float32Array.from(reversed.channels[0]!, (sample) => -sample));
76+
77+
it("still hears a channel that is the reverse of the other", () => {
78+
expect(moments(reversed)?.peak).toBeCloseTo(1.01, 2);
79+
for (const method of ["spectral-flux", "energy"] as const)
80+
expect(
81+
onsets(reversed, { method }).events.some((event) => Math.abs(event.t - 1) < 0.02),
82+
).toBe(true);
83+
expect(Math.max(...envelope(reversed, { hopMs: 10 }).values)).toBeGreaterThan(-40);
84+
expect(silence(reversed).some((span) => span.start <= 1.01 && span.end > 1.05)).toBe(false);
85+
});
86+
});
87+
7388
describe("onsets", () => {
7489
it("finds each hit of a three-hit effect, the loudest strongest, by spectral flux and by energy", () => {
7590
const hits = [0.2, 0.55, 0.9];

‎packages/core/src/audioAnalysis.ts‎

Lines changed: 41 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -120,14 +120,17 @@ const DB_FLOOR = -120;
120120
export const signalDuration = (signal: AudioSignal): number =>
121121
(signal.channels[0]?.length ?? 0) / signal.sampleRate;
122122

123-
export function mixToMono(signal: AudioSignal): Float32Array {
123+
/** Per-sample energy. Channels are squared first, so a reversed channel stays audible. */
124+
function energyMix(signal: AudioSignal): Float32Array {
124125
const [first, ...rest] = signal.channels;
125126
if (!first) return new Float32Array(0);
126127
if (!rest.length) return first;
127-
const mono = new Float32Array(first.length);
128+
const mixed = new Float32Array(first.length);
128129
for (const channel of signal.channels)
129-
for (let i = 0; i < mono.length; i++) mono[i]! += channel[i]! / signal.channels.length;
130-
return mono;
130+
for (let i = 0; i < mixed.length; i++) mixed[i]! += channel[i]! * channel[i]!;
131+
const scale = 1 / signal.channels.length;
132+
for (let i = 0; i < mixed.length; i++) mixed[i] = Math.sqrt(mixed[i]! * scale);
133+
return mixed;
131134
}
132135

133136
const toDb = (amplitude: number) =>
@@ -207,40 +210,60 @@ function spectra(
207210
return frames;
208211
}
209212

213+
function averageMagnitudes(frames: Float64Array[]): Float64Array {
214+
const mixed = new Float64Array(frames[0]?.length ?? 0);
215+
for (const frame of frames)
216+
for (let bin = 0; bin < mixed.length; bin++) mixed[bin]! += frame[bin]!;
217+
const scale = frames.length || 1;
218+
for (let bin = 0; bin < mixed.length; bin++) mixed[bin]! /= scale;
219+
return mixed;
220+
}
221+
222+
/** Mean magnitude spectrum across channels, so opposite polarity cannot cancel a bin. */
223+
function meanSpectra(signal: AudioSignal, hopMs: number, band?: [number, number]): Float64Array[] {
224+
const stacked = signal.channels.map((channel) =>
225+
spectra(channel, signal.sampleRate, hopMs, band),
226+
);
227+
const count = stacked[0]?.length ?? 0;
228+
const mixed: Float64Array[] = [];
229+
for (let frame = 0; frame < count; frame++)
230+
mixed.push(averageMagnitudes(stacked.map((channel) => channel[frame]!)));
231+
return mixed;
232+
}
233+
210234
export function envelope(signal: AudioSignal, options: EnvelopeOptions = {}): Envelope {
211235
const { hopMs = 5, scale = "db", band } = options;
212236
const windowMs = options.windowMs ?? hopMs * 2;
213-
const mono = mixToMono(signal);
214237
const size = fftSizeFor(signal.sampleRate);
215238
// Parseval over the band's bins; a Hann window keeps 0.375 of a frame's power.
216239
const amplitudes = band
217-
? spectra(mono, signal.sampleRate, hopMs, band).map((frame) => {
240+
? meanSpectra(signal, hopMs, band).map((frame) => {
218241
let sum = 0;
219242
for (const magnitude of frame) sum += magnitude * magnitude;
220243
return Math.sqrt((2 * sum) / (size * size * 0.375));
221244
})
222-
: rmsFrames(mono, signal.sampleRate, hopMs, windowMs);
245+
: rmsFrames(energyMix(signal), signal.sampleRate, hopMs, windowMs);
223246
return { hopMs, scale, values: amplitudes.map((a) => (scale === "db" ? toDb(a) : a)) };
224247
}
225248

226249
/** Positive change between neighbouring frames: spectral flux of log magnitudes, or the rise in RMS. */
227250
function noveltyCurve(
228-
mono: Float32Array,
229-
sampleRate: number,
251+
signal: AudioSignal,
230252
method: OnsetMethod,
231253
hopMs: number,
232254
band?: [number, number],
233255
): number[] {
256+
const samples = energyMix(signal);
234257
if (method === "energy") {
235-
const rms = rmsFrames(mono, sampleRate, hopMs, hopMs * 2);
258+
const rms = rmsFrames(samples, signal.sampleRate, hopMs, hopMs * 2);
236259
return rms.map((value, i) => Math.max(0, value - (rms[i - 1] ?? 0)));
237260
}
238-
const frames = spectra(mono, sampleRate, hopMs, band).map((frame) =>
261+
const frames = meanSpectra(signal, hopMs, band).map((frame) =>
239262
frame.map((m) => Math.log1p(100 * m)),
240263
);
241264
// A frame cut off by the end of the signal splatters across the spectrum; that is the edge, not a hit.
242265
const lastWhole = Math.floor(
243-
(mono.length - fftSizeFor(sampleRate) / 2) / hopSamples(sampleRate, hopMs),
266+
(samples.length - fftSizeFor(signal.sampleRate) / 2) / hopSamples(signal.sampleRate, hopMs),
244267
);
245268
return frames.map((frame, i) => {
246269
const previous = frames[i - 1];
@@ -273,7 +296,7 @@ function isOnsetPeak(
273296

274297
export function onsets(signal: AudioSignal, options: OnsetOptions = {}): Onsets {
275298
const { method = "spectral-flux", band, threshold = 0.1, minGapMs = 30, hopMs = 5 } = options;
276-
const curve = noveltyCurve(mixToMono(signal), signal.sampleRate, method, hopMs, band);
299+
const curve = noveltyCurve(signal, method, hopMs, band);
277300
const top = Math.max(0, ...curve);
278301
if (top <= 0) return { method, events: [] };
279302
const gap = Math.max(1, Math.round(minGapMs / hopMs));
@@ -293,7 +316,7 @@ export function onsets(signal: AudioSignal, options: OnsetOptions = {}): Onsets
293316

294317
export function moments(signal: AudioSignal, options: MomentOptions = {}): Moments | null {
295318
const { onsetRatio = 0.2, tailDb = -40, hopMs = 1 } = options;
296-
const rms = rmsFrames(mixToMono(signal), signal.sampleRate, hopMs, Math.max(4, hopMs * 4));
319+
const rms = rmsFrames(energyMix(signal), signal.sampleRate, hopMs, Math.max(4, hopMs * 4));
297320
const hopSeconds = hopSamples(signal.sampleRate, hopMs) / signal.sampleRate;
298321
let peakIndex = 0;
299322
for (let i = 1; i < rms.length; i++) if (rms[i]! > rms[peakIndex]!) peakIndex = i;
@@ -475,7 +498,7 @@ export function loudness(signal: AudioSignal): Loudness {
475498

476499
export function silence(signal: AudioSignal, options: SilenceOptions = {}): Span[] {
477500
const { thresholdDb = -50, minMs = 200, hopMs = 5 } = options;
478-
const db = rmsFrames(mixToMono(signal), signal.sampleRate, hopMs, hopMs * 2).map(toDb);
501+
const db = rmsFrames(energyMix(signal), signal.sampleRate, hopMs, hopMs * 2).map(toDb);
479502
const hopSeconds = hopSamples(signal.sampleRate, hopMs) / signal.sampleRate;
480503
const spans: Span[] = [];
481504
let from: number | null = null;
@@ -516,7 +539,7 @@ function periodicity(curve: number[], hopMs: number): number {
516539
/** How much of the envelope's modulation sits at syllable rate (3–8 Hz). */
517540
function syllableModulation(signal: AudioSignal): number {
518541
const hopMs = 10;
519-
const rms = rmsFrames(mixToMono(signal), signal.sampleRate, hopMs, 25);
542+
const rms = rmsFrames(energyMix(signal), signal.sampleRate, hopMs, 25);
520543
const size = 2 ** Math.floor(Math.log2(Math.min(rms.length, 1024)));
521544
if (size < 64) return 0;
522545
const re = Float64Array.from(rms.slice(0, size));
@@ -537,7 +560,7 @@ function syllableModulation(signal: AudioSignal): number {
537560
}
538561

539562
function sustainShare(signal: AudioSignal): number {
540-
const levels = rmsFrames(mixToMono(signal), signal.sampleRate, 50, 100).map(toDb);
563+
const levels = rmsFrames(energyMix(signal), signal.sampleRate, 50, 100).map(toDb);
541564
if (!levels.length) return 0;
542565
const loud = [...levels].sort((a, b) => a - b)[Math.floor((levels.length - 1) * 0.95)]!;
543566
return levels.filter((level) => level > loud - 20).length / levels.length;
@@ -546,9 +569,7 @@ function sustainShare(signal: AudioSignal): number {
546569
export function kindFeatures(signal: AudioSignal): KindFeatures {
547570
return {
548571
duration: round(signalDuration(signal)),
549-
regularity: round(
550-
periodicity(noveltyCurve(mixToMono(signal), signal.sampleRate, "spectral-flux", 10), 10),
551-
),
572+
regularity: round(periodicity(noveltyCurve(signal, "spectral-flux", 10), 10)),
552573
syllables: round(syllableModulation(signal)),
553574
sustain: round(sustainShare(signal)),
554575
};

0 commit comments

Comments
 (0)