Skip to content

Commit 8b81201

Browse files
klopez4212Mongo
andauthored
Fix mobile Huddle agent voice turn states (#6611)
## Summary - ignore mobile room tone when deciding whether human speech should interrupt agent audio - show a subtle waiting animation while an agent prepares a response, then restore its avatar when speech begins - render agents from Huddle membership before their audio peer starts transmitting ## Testing - `just ci` - signed Release build installed and launched on a physical iPhone - live Huddle voice and agent-membership behavior verified on device --------- Signed-off-by: kenny lopez <[email protected]> Co-authored-by: Mongo <9cfd347903944d5b85aa6c93d2ab67381b978a92a31914bca69998968752a1d7@buzz.block.builderlab.xyz>
1 parent 9aa332a commit 8b81201

9 files changed

Lines changed: 1060 additions & 197 deletions

File tree

‎desktop/src-tauri/src/huddle/playout.rs‎

Lines changed: 73 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,9 @@ const SPEAKER_LEVEL_TICK_MS: u64 = 50;
4444
/// Per-peer arrival window for the TTS interrupt frame counter.
4545
const FRAME_WINDOW: std::time::Duration = std::time::Duration::from_millis(500);
4646
const REMOTE_RELEASE_DEBOUNCE: std::time::Duration = std::time::Duration::from_millis(500);
47+
/// Match Mobile's speaking treatment: an open microphone can emit continuous
48+
/// non-DTX Opus for room tone, so packet type alone is not evidence of speech.
49+
const REMOTE_SPEECH_LEVEL_DBOV: i8 = -55;
4750
/// Playout clock: NetEq emits 10 ms frames, so we tick at 10 ms.
4851
const PLAYOUT_TICK_MS: u64 = 10;
4952

@@ -86,19 +89,30 @@ fn normalized_speaker_level(level_dbov: i8) -> f32 {
8689
((f32::from(level_dbov) + 60.0) / 48.0).clamp(0.0, 1.0)
8790
}
8891

92+
fn is_remote_speech_frame(is_dtx: bool, level_dbov: i8) -> bool {
93+
!is_dtx && level_dbov >= REMOTE_SPEECH_LEVEL_DBOV
94+
}
95+
8996
fn update_remote_release_deadline(
9097
peer: u8,
91-
is_dtx: bool,
98+
is_speech: bool,
9299
remote_floor_owners: &std::collections::HashSet<u8>,
93100
deadlines: &mut std::collections::HashMap<u8, tokio::time::Instant>,
94101
now: tokio::time::Instant,
95102
) {
96-
if !is_dtx {
97-
deadlines.remove(&peer);
98-
} else if remote_floor_owners.contains(&peer) {
99-
deadlines
100-
.entry(peer)
101-
.or_insert(now + REMOTE_RELEASE_DEBOUNCE);
103+
if remote_floor_owners.contains(&peer) {
104+
if is_speech {
105+
// Refresh from audible speech itself. Some mobile capture paths
106+
// stop producing packets once speech ends, so waiting for a DTX
107+
// or quiet packet can otherwise hold the human floor forever.
108+
deadlines.insert(peer, now + REMOTE_RELEASE_DEBOUNCE);
109+
} else {
110+
// Preserve the deadline from the last audible frame. Continuous
111+
// room-tone packets must not keep extending the human floor.
112+
deadlines
113+
.entry(peer)
114+
.or_insert(now + REMOTE_RELEASE_DEBOUNCE);
115+
}
102116
}
103117
}
104118

@@ -437,19 +451,20 @@ pub(crate) async fn run_playout_recv_loop(
437451
continue;
438452
}
439453
let is_dtx = (header.flags & FLAG_DTX) != 0;
440-
// Only count non-DTX arrivals toward the UI's
441-
// active-speaker set. DTX/comfort packets are emitted
442-
// by an idle peer to keep the codec alive — they
443-
// don't mean the peer is speaking, and shouldn't
444-
// make their tile flash for the 500 ms speaker tick.
454+
let is_remote_speech =
455+
is_remote_speech_frame(is_dtx, header.level_dbov);
456+
// Only count audible arrivals toward the UI's
457+
// active-speaker set. An open mobile microphone can
458+
// continuously emit non-DTX room tone, so require an
459+
// audible level before treating a packet as speech.
445460
update_remote_release_deadline(
446461
peer_idx,
447-
is_dtx,
462+
is_remote_speech,
448463
&remote_floor_owners,
449464
&mut remote_release_deadlines,
450465
tokio::time::Instant::now(),
451466
);
452-
if !is_dtx {
467+
if is_remote_speech {
453468
active_indices.insert(peer_idx);
454469
let level = normalized_speaker_level(header.level_dbov);
455470
speaker_levels
@@ -497,7 +512,7 @@ pub(crate) async fn run_playout_recv_loop(
497512
// Count only remote-human speech toward floor onset.
498513
// Agent audio still plays, but it must not acquire the
499514
// human floor or suppress another agent's response.
500-
if !is_dtx && remote_human {
515+
if is_remote_speech && remote_human {
501516
if last_frame_reset.elapsed() >= FRAME_WINDOW {
502517
frame_counts.clear();
503518
last_frame_reset = tokio::time::Instant::now();
@@ -507,6 +522,14 @@ pub(crate) async fn run_playout_recv_loop(
507522
if *count >= REMOTE_SPEECH_THRESHOLD {
508523
human_floor.enter_remote(peer_idx);
509524
remote_floor_owners.insert(peer_idx);
525+
// The threshold-crossing frame is processed
526+
// before this peer becomes an owner. Arm its
527+
// release here so silence need not arrive in a
528+
// later packet to let queued TTS continue.
529+
remote_release_deadlines.insert(
530+
peer_idx,
531+
tokio::time::Instant::now() + REMOTE_RELEASE_DEBOUNCE,
532+
);
510533
if tts_active.load(Ordering::Acquire) {
511534
tts_cancel.store(true, Ordering::Release);
512535
}
@@ -647,18 +670,18 @@ mod tests {
647670
use super::*;
648671

649672
#[test]
650-
fn continuous_dtx_does_not_extend_remote_floor_deadline() {
673+
fn continuous_silence_does_not_extend_remote_floor_deadline() {
651674
let peer = 7;
652675
let started = tokio::time::Instant::now();
653676
let owners = std::collections::HashSet::from([peer]);
654677
let mut deadlines = std::collections::HashMap::new();
655678

656-
update_remote_release_deadline(peer, true, &owners, &mut deadlines, started);
679+
update_remote_release_deadline(peer, false, &owners, &mut deadlines, started);
657680
let armed = deadlines[&peer];
658681
for elapsed_ms in [100, 200, 300, 400] {
659682
update_remote_release_deadline(
660683
peer,
661-
true,
684+
false,
662685
&owners,
663686
&mut deadlines,
664687
started + std::time::Duration::from_millis(elapsed_ms),
@@ -678,18 +701,48 @@ mod tests {
678701
}
679702

680703
#[test]
681-
fn dtx_from_non_owner_does_not_arm_remote_floor_deadline() {
704+
fn last_speech_frame_arms_remote_floor_release_without_follow_up_audio() {
705+
let peer = 7;
706+
let started = tokio::time::Instant::now();
707+
let owners = std::collections::HashSet::from([peer]);
708+
let mut deadlines = std::collections::HashMap::new();
709+
710+
update_remote_release_deadline(peer, true, &owners, &mut deadlines, started);
711+
let armed = started + REMOTE_RELEASE_DEBOUNCE;
712+
assert_eq!(deadlines[&peer], armed);
713+
714+
let human_floor = HumanFloor::new();
715+
human_floor.enter_remote(peer);
716+
let mut owners = owners;
717+
release_expired_remote_floors(armed, &mut owners, &mut deadlines, &human_floor);
718+
719+
assert!(!human_floor.is_blocked());
720+
assert!(owners.is_empty());
721+
assert!(deadlines.is_empty());
722+
}
723+
724+
#[test]
725+
fn silence_from_non_owner_does_not_arm_remote_floor_deadline() {
682726
let mut deadlines = std::collections::HashMap::new();
683727
update_remote_release_deadline(
684728
7,
685-
true,
729+
false,
686730
&std::collections::HashSet::new(),
687731
&mut deadlines,
688732
tokio::time::Instant::now(),
689733
);
690734
assert!(deadlines.is_empty());
691735
}
692736

737+
#[test]
738+
fn remote_speech_requires_non_dtx_audio_above_the_activity_floor() {
739+
assert!(!is_remote_speech_frame(true, 0));
740+
assert!(!is_remote_speech_frame(false, -127));
741+
assert!(!is_remote_speech_frame(false, -56));
742+
assert!(is_remote_speech_frame(false, -55));
743+
assert!(is_remote_speech_frame(false, -12));
744+
}
745+
693746
#[test]
694747
fn speaker_level_maps_conversational_range() {
695748
assert_eq!(normalized_speaker_level(-127), 0.0);

‎mobile/lib/features/channels/channel_detail_page.dart‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import '../../shared/relay/relay.dart';
1919
import '../../shared/theme/theme.dart';
2020
import '../../shared/widgets/avatar_image.dart';
2121
import '../../shared/widgets/buzz_loading_indicator.dart';
22+
import '../../shared/widgets/bouncing_dots_indicator.dart';
2223
import '../../shared/widgets/concentric_sheet_surface.dart';
2324
import '../../shared/widgets/frosted_app_bar.dart';
2425
import '../../shared/widgets/frosted_scaffold.dart';

‎mobile/lib/features/channels/channel_detail_page/huddle_call_avatar.dart‎

Lines changed: 111 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,15 @@
11
part of '../channel_detail_page.dart';
22

3+
const _huddleAgentResponseResetDelay = Duration(milliseconds: 1200);
4+
35
class _HuddleCallAvatar extends HookConsumerWidget {
46
const _HuddleCallAvatar({
57
required this.pubkey,
68
required this.profile,
79
required this.fallbackLabel,
810
required this.active,
911
required this.speakerLevel,
12+
required this.preparingResponse,
1013
required this.onTap,
1114
this.isSelf = false,
1215
this.frameSize = _huddleAvatarFrameSize,
@@ -17,13 +20,70 @@ class _HuddleCallAvatar extends HookConsumerWidget {
1720
final String? fallbackLabel;
1821
final bool active;
1922
final double speakerLevel;
23+
final bool preparingResponse;
2024
final VoidCallback? onTap;
2125
final bool isSelf;
2226
final double frameSize;
2327

2428
@override
2529
Widget build(BuildContext context, WidgetRef ref) {
2630
final reducedMotion = MediaQuery.disableAnimationsOf(context);
31+
final responseHasStarted = useState(false);
32+
final responseResetTimer = useRef<Timer?>(null);
33+
// Tracks the previous working signal so we can detect its edges, and
34+
// whether the working signal has cleared since audio latched the response.
35+
// A completed working cycle distinguishes a genuinely new turn from late
36+
// typing that merely trails the turn already spoken.
37+
final wasPreparing = useRef(false);
38+
final workingCycleCompleted = useRef(false);
39+
useEffect(() {
40+
final preparingStarted = preparingResponse && !wasPreparing.value;
41+
final preparingCleared = !preparingResponse && wasPreparing.value;
42+
wasPreparing.value = preparingResponse;
43+
if (active) {
44+
// Audio for the current turn latches that a response has begun and
45+
// starts a fresh turn, so any prior working cycle no longer applies.
46+
responseResetTimer.value?.cancel();
47+
responseResetTimer.value = null;
48+
responseHasStarted.value = true;
49+
workingCycleCompleted.value = false;
50+
} else if (preparingStarted &&
51+
responseHasStarted.value &&
52+
workingCycleCompleted.value) {
53+
// A new working turn began after the previous turn's working signal
54+
// already cleared, so the "already spoke" suppression no longer
55+
// applies — allow the preparing indicator to show again.
56+
responseResetTimer.value?.cancel();
57+
responseResetTimer.value = null;
58+
responseHasStarted.value = false;
59+
workingCycleCompleted.value = false;
60+
} else if (preparingResponse) {
61+
// Working signal (including late typing for the turn just spoken) holds
62+
// the suppression alive; keep the reset timer cancelled.
63+
responseResetTimer.value?.cancel();
64+
responseResetTimer.value = null;
65+
} else if (responseHasStarted.value && responseResetTimer.value == null) {
66+
responseResetTimer.value = Timer(_huddleAgentResponseResetDelay, () {
67+
responseResetTimer.value = null;
68+
responseHasStarted.value = false;
69+
workingCycleCompleted.value = false;
70+
});
71+
}
72+
// Record that this turn's working signal has completed a cycle once it
73+
// clears after audio latched, so the next working turn is not mistaken
74+
// for trailing typing.
75+
if (preparingCleared && responseHasStarted.value) {
76+
workingCycleCompleted.value = true;
77+
}
78+
return null;
79+
}, [active, preparingResponse, responseHasStarted.value]);
80+
useEffect(
81+
() =>
82+
() => responseResetTimer.value?.cancel(),
83+
const [],
84+
);
85+
final showPreparingResponse =
86+
preparingResponse && !active && !responseHasStarted.value;
2787
final scale = frameSize / _huddleAvatarFrameSize;
2888
final avatarRadius = _huddleAvatarRadius * scale;
2989
final speakingRingSize = _huddleSpeakingRingSize * scale;
@@ -65,12 +125,22 @@ class _HuddleCallAvatar extends HookConsumerWidget {
65125
isSelf: isSelf,
66126
);
67127

128+
final semanticStates = [
129+
label,
130+
if (showPreparingResponse) 'preparing a response',
131+
if (active) 'speaking',
132+
].join(', ');
133+
68134
return SizedBox(
69135
width: frameSize,
70136
child: Semantics(
71-
label: active ? '$label, speaking' : label,
137+
label: semanticStates,
72138
hint: onTap == null ? null : 'Tap to focus participant',
73139
button: onTap != null,
140+
// The outer node excludes descendant semantics, so the child
141+
// indicator's live region never reaches assistive tech. Promote this
142+
// node to a live region while preparing so the label change announces.
143+
liveRegion: showPreparingResponse,
74144
onTap: onTap,
75145
excludeSemantics: true,
76146
child: GestureDetector(
@@ -121,15 +191,46 @@ class _HuddleCallAvatar extends HookConsumerWidget {
121191
),
122192
),
123193
),
124-
AvatarImage(
125-
imageUrl: profile?.avatarUrl,
126-
radius: avatarRadius,
127-
backgroundColor: context.colors.primaryContainer,
128-
fallback: Icon(
129-
LucideIcons.userRound,
130-
size: fallbackIconSize,
131-
color: context.colors.onPrimaryContainer,
132-
),
194+
AnimatedSwitcher(
195+
duration: reducedMotion
196+
? Duration.zero
197+
: const Duration(milliseconds: 180),
198+
switchInCurve: Curves.easeOutCubic,
199+
switchOutCurve: Curves.easeInCubic,
200+
transitionBuilder: (child, animation) =>
201+
FadeTransition(opacity: animation, child: child),
202+
child: showPreparingResponse
203+
? Container(
204+
key: ValueKey(
205+
'huddle-agent-preparing-response-$pubkey',
206+
),
207+
width: avatarRadius * 2,
208+
height: avatarRadius * 2,
209+
decoration: BoxDecoration(
210+
shape: BoxShape.circle,
211+
color: context.colors.primaryContainer,
212+
),
213+
alignment: Alignment.center,
214+
child: BouncingDotsIndicator(
215+
color: context.colors.onPrimaryContainer,
216+
dotSize: 6 * scale,
217+
gap: 4 * scale,
218+
semanticLabel:
219+
'$label is preparing a response',
220+
),
221+
)
222+
: AvatarImage(
223+
key: ValueKey('huddle-avatar-image-$pubkey'),
224+
imageUrl: profile?.avatarUrl,
225+
radius: avatarRadius,
226+
backgroundColor:
227+
context.colors.primaryContainer,
228+
fallback: Icon(
229+
LucideIcons.userRound,
230+
size: fallbackIconSize,
231+
color: context.colors.onPrimaryContainer,
232+
),
233+
),
133234
),
134235
],
135236
),

‎mobile/lib/features/channels/channel_detail_page/huddle_call_participants.dart‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ class _HuddleCallParticipants extends StatelessWidget {
1010
required this.localPubkey,
1111
required this.activeSpeakerPubkeys,
1212
required this.speakerLevels,
13+
required this.workingAgentPubkeys,
1314
required this.retryTooltip,
1415
required this.retryIcon,
1516
required this.onRetry,
@@ -25,6 +26,7 @@ class _HuddleCallParticipants extends StatelessWidget {
2526
final String? localPubkey;
2627
final Set<String> activeSpeakerPubkeys;
2728
final Map<String, double> speakerLevels;
29+
final Set<String> workingAgentPubkeys;
2830
final String retryTooltip;
2931
final IconData retryIcon;
3032
final VoidCallback onRetry;
@@ -126,6 +128,7 @@ class _HuddleCallParticipants extends StatelessWidget {
126128
speakerLevel: localPubkey == null
127129
? 0
128130
: speakerLevels[localPubkey] ?? 0,
131+
preparingResponse: false,
129132
isSelf: true,
130133
onTap: null,
131134
),
@@ -145,6 +148,7 @@ class _HuddleCallParticipants extends StatelessWidget {
145148
fallbackLabels: fallbackLabels,
146149
activeSpeakerPubkeys: activeSpeakerPubkeys,
147150
speakerLevels: speakerLevels,
151+
workingAgentPubkeys: workingAgentPubkeys,
148152
movementDuration: movementDuration,
149153
entryDuration: entryDuration,
150154
exitDuration: exitDuration,

0 commit comments

Comments
 (0)