Skip to content

Commit 114dbf7

Browse files
klopez4212Princess DonutMongo
authored
Add voice notes to desktop messages (#6978)
## Summary - add voice-note recording, preview, removal, and send controls to the desktop composer - render sent voice notes as waveform cards with scrubbing and playback-speed controls - transcode recordings locally into the existing relay video-media contract, without relay changes ## Testing - `just ci` - `pnpm exec playwright test tests/e2e/voice-note.spec.ts` - native voice-note media contract test against the relay validator ## Screenshots Focused snapshots are attached below. --------- Signed-off-by: kenny lopez <[email protected]> Signed-off-by: Princess Donut <b238ea756dee4d98afa5883fc7f1de61eeabe65bf700e3a5a5a80db5e42e2c2b@buzz.block.builderlab.xyz> Co-authored-by: Princess Donut <b238ea756dee4d98afa5883fc7f1de61eeabe65bf700e3a5a5a80db5e42e2c2b@buzz.block.builderlab.xyz> Co-authored-by: Mongo <9cfd347903944d5b85aa6c93d2ab67381b978a92a31914bca69998968752a1d7@buzz.block.builderlab.xyz>
1 parent 59328d5 commit 114dbf7

47 files changed

Lines changed: 3956 additions & 153 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎desktop/playwright.config.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ export default defineConfig({
5858
"**/welcome-agent-modal-screenshots.spec.ts",
5959
"**/local-archive-screenshots.spec.ts",
6060
"**/voice-settings.spec.ts",
61+
"**/voice-note.spec.ts",
6162
"**/agent-readiness-screenshots.spec.ts",
6263
"**/agent-error-state-screenshots.spec.ts",
6364
"**/edit-agent.spec.ts",

‎desktop/src-tauri/Cargo.toml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,6 @@ tauri-utils = "2"
154154
# `test-util` enables tokio's paused-clock (`start_paused`) so the relay
155155
# admission gate tests can assert exact wait durations without real sleeps.
156156
tokio = { version = "1", features = ["test-util"] }
157-
# The relay's media validation, so the snapshot-sharing tests can prove the
158-
# full export → sanitize → relay-accept → import contract end to end.
157+
# The relay's media validation, so desktop-produced snapshots and voice notes
158+
# can prove their full client-sanitize → relay-accept contract end to end.
159159
buzz_media_pkg = { package = "buzz-media", path = "../../crates/buzz-media" }

‎desktop/src-tauri/Info.plist‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
<key>CFBundleName</key>
88
<string>Buzz</string>
99
<key>NSMicrophoneUsageDescription</key>
10-
<string>Buzz needs microphone access for voice huddles.</string>
10+
<string>Buzz needs microphone access for voice huddles and voice notes.</string>
1111
<key>NSCameraUsageDescription</key>
1212
<string>Buzz needs camera access to record animated avatars.</string>
1313
<key>NSLocalNetworkUsageDescription</key>

‎desktop/src-tauri/src/commands/media.rs‎

Lines changed: 17 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,16 @@ use tokio_util::sync::CancellationToken;
88
use crate::app_state::AppState;
99
use crate::relay::{parse_json_response, relay_api_base_url_with_override, relay_error_message};
1010

11+
use super::media_filename::sanitize_filename;
1112
use super::media_transcode::{
1213
has_heic_extension, is_heic_file, is_video_file, transcode_and_extract_poster,
1314
transcode_and_extract_poster_with_cancellation, transcode_heic_path_to_jpeg_bytes,
1415
transcode_heic_path_to_jpeg_bytes_with_cancellation,
1516
};
1617
use super::media_upload_progress::{emit_media_upload_phase, send_upload_attempt, UploadAttempt};
18+
use super::media_voice_note::{
19+
is_voice_note_filename, prepare_voice_note_for_upload, voice_note_mp4_filename,
20+
};
1721

1822
#[derive(Debug, Clone, Serialize, Deserialize)]
1923
pub struct BlobDescriptor {
@@ -134,24 +138,6 @@ const BLOCKED_MIME: &[&str] = &[
134138
"application/x-apple-diskimage",
135139
];
136140

137-
/// Sanitize a filename for use as a display label in the imeta `filename` field.
138-
///
139-
/// Strips any directory components (keeps only the final path segment), removes
140-
/// control characters, and bounds length to 255. Mirrors the relay's filename
141-
/// validation so a sanitized name always passes ingest. Returns a fallback when
142-
/// the result would be empty.
143-
pub(crate) fn sanitize_filename(name: &str) -> String {
144-
// Keep only the final path segment — defend against `../` and absolute paths
145-
// regardless of separator style.
146-
let base = name.rsplit(['/', '\\']).next().unwrap_or(name).trim();
147-
let cleaned: String = base.chars().filter(|c| !c.is_control()).take(255).collect();
148-
if cleaned.is_empty() {
149-
"file".to_string()
150-
} else {
151-
cleaned
152-
}
153-
}
154-
155141
/// Return true when a PNG/WebP payload declares animation.
156142
///
157143
/// Animated payloads use structural sanitizers so frame timing, looping, and
@@ -724,8 +710,12 @@ pub(super) async fn upload_media_bytes_inner(
724710
let heic_by_extension = filename
725711
.as_deref()
726712
.is_some_and(|name| has_heic_extension(std::path::Path::new(name)));
713+
let is_voice_note = is_voice_note_filename(filename.as_deref());
727714

728-
let (body, poster_bytes) = if is_video_file(&data) {
715+
let (body, poster_bytes) = if is_voice_note {
716+
emit_media_upload_phase(&app, progress_id.as_deref(), "processing-audio");
717+
prepare_voice_note_for_upload(data, cancellation).await?
718+
} else if is_video_file(&data) {
729719
emit_media_upload_phase(&app, progress_id.as_deref(), "processing-video");
730720
// Video: write to temp → transcode + extract poster → read results.
731721
// All blocking I/O runs off the async runtime via spawn_blocking.
@@ -790,7 +780,14 @@ pub(super) async fn upload_media_bytes_inner(
790780
}
791781
}
792782

793-
descriptor.filename = filename.as_deref().map(sanitize_filename);
783+
descriptor.filename = filename.as_deref().map(|name| {
784+
let upload_name = if is_voice_note {
785+
voice_note_mp4_filename(name)
786+
} else {
787+
name.to_string()
788+
};
789+
sanitize_filename(&upload_name)
790+
});
794791

795792
Ok(descriptor)
796793
}
@@ -981,18 +978,4 @@ mod tests {
981978
reqwest::StatusCode::UNSUPPORTED_MEDIA_TYPE
982979
));
983980
}
984-
985-
#[test]
986-
fn test_sanitize_filename() {
987-
assert_eq!(sanitize_filename("report.pdf"), "report.pdf");
988-
// Strips directory components and traversal.
989-
assert_eq!(sanitize_filename("../../etc/passwd"), "passwd");
990-
assert_eq!(sanitize_filename("/abs/path/notes.txt"), "notes.txt");
991-
assert_eq!(sanitize_filename(r"C:\Users\me\doc.docx"), "doc.docx");
992-
// Empty / separator-only falls back.
993-
assert_eq!(sanitize_filename(""), "file");
994-
assert_eq!(sanitize_filename("/"), "file");
995-
// Control chars removed.
996-
assert_eq!(sanitize_filename("a\nb\tc.txt"), "abc.txt");
997-
}
998981
}

‎desktop/src-tauri/src/commands/media_download.rs‎

Lines changed: 31 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,13 @@
11
use futures_util::StreamExt;
22
use sha2::{Digest, Sha256};
33
use tauri::State;
4+
use tokio_util::sync::CancellationToken;
45

56
use crate::app_state::AppState;
67
use crate::commands::clipboard::with_clipboard;
78
use crate::commands::export_util::save_bytes_with_dialog;
8-
use crate::commands::media::{detect_and_validate_mime, mint_media_get_auth, sanitize_filename};
9+
use crate::commands::media::{detect_and_validate_mime, mint_media_get_auth};
10+
use crate::commands::media_filename::sanitize_filename;
911
use crate::commands::{
1012
personas::{
1113
parse_snapshot_payload_from_bytes, MAX_SNAPSHOT_JSON_BYTES, MAX_SNAPSHOT_PNG_BYTES,
@@ -18,7 +20,7 @@ use crate::commands::{
1820
use crate::relay::{classify_request_error, relay_api_base_url_with_override, relay_error_message};
1921

2022
/// Maximum download size: 50 MiB. Prevents OOM from oversized responses.
21-
const MAX_DOWNLOAD_BYTES: u64 = 50 * 1024 * 1024;
23+
pub(super) const MAX_DOWNLOAD_BYTES: u64 = 50 * 1024 * 1024;
2224

2325
/// Download request timeout.
2426
const DOWNLOAD_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60);
@@ -29,7 +31,7 @@ const DOWNLOAD_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60)
2931
/// - URL scheme is `https` (or `http` for localhost dev)
3032
/// - URL origin matches the relay base URL
3133
/// - URL path matches `/media/{hash}.{ext}`
32-
fn validate_download_url(url: &str, relay_base: &str) -> Result<(), String> {
34+
pub(super) fn validate_download_url(url: &str, relay_base: &str) -> Result<(), String> {
3335
let parsed = url::Url::parse(url).map_err(|_| "invalid URL".to_string())?;
3436
let base = url::Url::parse(relay_base).map_err(|_| "invalid relay base URL".to_string())?;
3537

@@ -139,32 +141,6 @@ pub async fn download_file(
139141
save_bytes_with_dialog(&app, &filename, "All Files", &extensions, &bytes).await
140142
}
141143

142-
/// Fetch relay media bytes for the composer image editor.
143-
///
144-
/// The editor composites the image onto a canvas and needs pixel access.
145-
/// Handing the webview raw bytes over IPC (which it wraps in a same-origin
146-
/// `blob:` URL) keeps the canvas un-tainted without involving CORS — and
147-
/// therefore without any media-proxy header or origin-gate changes.
148-
///
149-
/// Same SSRF validation, size cap, and content policy as the download
150-
/// commands above.
151-
///
152-
/// Returns `tauri::ipc::Response` so the bytes cross IPC as a raw buffer
153-
/// instead of a JSON number array (which would be ~3x the size to
154-
/// serialize and deserialize at the 50 MiB cap).
155-
#[tauri::command]
156-
pub async fn fetch_media_bytes(
157-
url: String,
158-
state: State<'_, AppState>,
159-
) -> Result<tauri::ipc::Response, String> {
160-
let relay_base = relay_api_base_url_with_override(&state);
161-
validate_download_url(&url, &relay_base)?;
162-
163-
let bytes = fetch_blob_bytes(&url, &state).await?;
164-
detect_and_validate_mime(&bytes)?;
165-
Ok(tauri::ipc::Response::new(bytes))
166-
}
167-
168144
/// Copy an image from a relay media URL directly to the system clipboard.
169145
///
170146
/// Fetches the image, decodes it to RGBA8, and writes it to the clipboard via
@@ -255,7 +231,7 @@ pub async fn copy_text_to_clipboard(
255231
/// HTTP client, enforcing the download size cap. The caller is responsible for
256232
/// validating the URL origin and for any content-type checks on the result.
257233
async fn fetch_blob_bytes(url: &str, state: &State<'_, AppState>) -> Result<Vec<u8>, String> {
258-
fetch_blob_bytes_with_cap(url, state, MAX_DOWNLOAD_BYTES).await
234+
fetch_blob_bytes_with_cap(url, state, MAX_DOWNLOAD_BYTES, None).await
259235
}
260236

261237
/// The command-facing error for a media-fetch response status, or `None` if
@@ -277,10 +253,11 @@ fn redirect_refusal_error(status: reqwest::StatusCode) -> Option<String> {
277253
}
278254

279255
/// Core streaming fetcher with a caller-supplied byte cap.
280-
async fn fetch_blob_bytes_with_cap(
256+
pub(super) async fn fetch_blob_bytes_with_cap(
281257
url: &str,
282258
state: &State<'_, AppState>,
283259
cap: u64,
260+
cancellation: Option<&CancellationToken>,
284261
) -> Result<Vec<u8>, String> {
285262
// Fetch bytes via the no-redirect media client (goes through the VPN tunnel).
286263
// A no-redirect client keeps the minted media auth token from being
@@ -296,7 +273,16 @@ async fn fetch_blob_bytes_with_cap(
296273
req = req.header("authorization", auth);
297274
}
298275

299-
let resp = req.send().await.map_err(|e| classify_request_error(&e))?;
276+
let request = req.send();
277+
let resp = if let Some(cancellation) = cancellation {
278+
tokio::select! {
279+
_ = cancellation.cancelled() => return Err("media fetch cancelled".to_string()),
280+
result = request => result,
281+
}
282+
} else {
283+
request.await
284+
}
285+
.map_err(|e| classify_request_error(&e))?;
300286

301287
if let Some(err) = redirect_refusal_error(resp.status()) {
302288
return Err(err);
@@ -321,7 +307,18 @@ async fn fetch_blob_bytes_with_cap(
321307
// even when Content-Length is missing or dishonest.
322308
let mut bytes = Vec::new();
323309
let mut stream = resp.bytes_stream();
324-
while let Some(chunk) = stream.next().await {
310+
loop {
311+
let next = if let Some(cancellation) = cancellation {
312+
tokio::select! {
313+
_ = cancellation.cancelled() => return Err("media fetch cancelled".to_string()),
314+
next = stream.next() => next,
315+
}
316+
} else {
317+
stream.next().await
318+
};
319+
let Some(chunk) = next else {
320+
break;
321+
};
325322
let chunk = chunk.map_err(|e| classify_request_error(&e))?;
326323
if bytes.len() as u64 + chunk.len() as u64 > cap {
327324
return Err(format!("file too large (max {} MiB)", cap / (1024 * 1024)));
@@ -482,7 +479,7 @@ pub async fn fetch_snapshot_bytes(
482479
ensure_declared_size_within_cap(expected_size, kind)?;
483480

484481
// ── Bounded fetch ─────────────────────────────────────────────────────
485-
let bytes = fetch_blob_bytes_with_cap(&url, &state, cap).await?;
482+
let bytes = fetch_blob_bytes_with_cap(&url, &state, cap, None).await?;
486483

487484
// ── Post-fetch validation ─────────────────────────────────────────────
488485
// 1. Byte length must equal the declared imeta size.
Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
use std::{
2+
collections::HashMap,
3+
sync::{LazyLock, Mutex},
4+
};
5+
6+
use tokio_util::sync::CancellationToken;
7+
8+
use crate::app_state::AppState;
9+
use crate::commands::media::detect_and_validate_mime;
10+
use crate::commands::media_download::{
11+
fetch_blob_bytes_with_cap, validate_download_url, MAX_DOWNLOAD_BYTES,
12+
};
13+
use crate::relay::relay_api_base_url_with_override;
14+
15+
#[derive(Default)]
16+
struct MediaFetchCancellations {
17+
tokens: HashMap<String, CancellationToken>,
18+
}
19+
20+
impl MediaFetchCancellations {
21+
fn begin(&mut self, request_id: &str) -> CancellationToken {
22+
if let Some(cancel) = self.tokens.get(request_id).cloned() {
23+
return cancel;
24+
}
25+
let cancel = CancellationToken::new();
26+
self.tokens.insert(request_id.to_string(), cancel.clone());
27+
cancel
28+
}
29+
30+
fn cancel(&mut self, request_id: &str) {
31+
self.tokens
32+
.entry(request_id.to_string())
33+
.or_default()
34+
.cancel();
35+
}
36+
37+
fn finish(&mut self, request_id: &str) {
38+
self.tokens.remove(request_id);
39+
}
40+
}
41+
42+
static MEDIA_FETCH_CANCELLATIONS: LazyLock<Mutex<MediaFetchCancellations>> =
43+
LazyLock::new(|| Mutex::new(MediaFetchCancellations::default()));
44+
45+
pub(super) fn begin_media_fetch(request_id: Option<&str>) -> Option<CancellationToken> {
46+
let request_id = request_id?;
47+
MEDIA_FETCH_CANCELLATIONS
48+
.lock()
49+
.ok()
50+
.map(|mut fetches| fetches.begin(request_id))
51+
}
52+
53+
pub(super) fn finish_media_fetch(request_id: Option<&str>) {
54+
let Some(request_id) = request_id else {
55+
return;
56+
};
57+
if let Ok(mut fetches) = MEDIA_FETCH_CANCELLATIONS.lock() {
58+
fetches.finish(request_id);
59+
}
60+
}
61+
62+
/// Cancel a renderer-owned relay media fetch, including an in-flight body.
63+
#[tauri::command]
64+
pub fn cancel_media_fetch(request_id: String) {
65+
if let Ok(mut fetches) = MEDIA_FETCH_CANCELLATIONS.lock() {
66+
fetches.cancel(&request_id);
67+
}
68+
}
69+
70+
/// Release renderer ownership after the fetch promise settles.
71+
#[tauri::command]
72+
pub fn release_media_fetch(request_id: String) {
73+
finish_media_fetch(Some(&request_id));
74+
}
75+
76+
/// Fetch relay media bytes with renderer-owned cancellation.
77+
#[tauri::command]
78+
pub async fn fetch_media_bytes(
79+
url: String,
80+
request_id: Option<String>,
81+
state: tauri::State<'_, AppState>,
82+
) -> Result<tauri::ipc::Response, String> {
83+
let cancellation = begin_media_fetch(request_id.as_deref());
84+
let result = async {
85+
let relay_base = relay_api_base_url_with_override(&state);
86+
validate_download_url(&url, &relay_base)?;
87+
let bytes =
88+
fetch_blob_bytes_with_cap(&url, &state, MAX_DOWNLOAD_BYTES, cancellation.as_ref())
89+
.await?;
90+
detect_and_validate_mime(&bytes)?;
91+
Ok(tauri::ipc::Response::new(bytes))
92+
}
93+
.await;
94+
finish_media_fetch(request_id.as_deref());
95+
result
96+
}
97+
98+
#[cfg(test)]
99+
mod tests {
100+
use super::*;
101+
102+
#[test]
103+
fn cancellation_before_begin_is_retained() {
104+
let mut fetches = MediaFetchCancellations::default();
105+
fetches.cancel("cancel-before-begin");
106+
107+
let cancellation = fetches.begin("cancel-before-begin");
108+
109+
assert!(cancellation.is_cancelled());
110+
fetches.finish("cancel-before-begin");
111+
assert!(fetches.tokens.is_empty());
112+
}
113+
114+
#[test]
115+
fn cancellation_reaches_active_owner() {
116+
let mut fetches = MediaFetchCancellations::default();
117+
let cancellation = fetches.begin("active-fetch");
118+
119+
fetches.cancel("active-fetch");
120+
121+
assert!(cancellation.is_cancelled());
122+
fetches.finish("active-fetch");
123+
assert!(fetches.tokens.is_empty());
124+
}
125+
}

0 commit comments

Comments
 (0)