Skip to content

Commit 20816f6

Browse files
steipetesteipete
authored
feat(voice): universal talk waveform driven by real audio levels (#102901)
* feat(voice): universal talk waveform driven by real audio levels One shared Siri-style talk animation across iOS, watchOS, macOS, and Android (TalkWaveformView + exact Compose port), replacing per-platform fakes: synthetic speaking pulses, constant speech-detected power, static Android bar rows, flat realtime listening. Listening/recording follow live mic levels on every route; agent speech follows the real playback envelope (AVAudioPlayer metering + playback-aligned PCM envelope + WebRTC stats); voice-note recording shows a live capture wave. * fix(android): order waveform imports and sync native i18n inventory * chore(i18n): resync native inventory after rebase
1 parent 3c048ef commit 20816f6

38 files changed

Lines changed: 1873 additions & 642 deletions

‎apps/.i18n/native-source.json‎

Lines changed: 308 additions & 308 deletions
Large diffs are not rendered by default.

‎apps/android/app/src/main/java/ai/openclaw/app/MainViewModel.kt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -262,6 +262,9 @@ class MainViewModel(
262262
val talkModeEnabled: StateFlow<Boolean> = runtimeState(initial = false) { it.talkModeEnabled }
263263
val talkModeListening: StateFlow<Boolean> = runtimeState(initial = false) { it.talkModeListening }
264264
val talkModeSpeaking: StateFlow<Boolean> = runtimeState(initial = false) { it.talkModeSpeaking }
265+
val talkInputLevel: StateFlow<Float> = runtimeState(initial = 0f) { it.talkInputLevel }
266+
val talkOutputLevel: StateFlow<Float?> = runtimeState(initial = null) { it.talkOutputLevel }
267+
val talkSpeechActive: StateFlow<Boolean> = runtimeState(initial = false) { it.talkSpeechActive }
265268
val talkModeStatusText: StateFlow<String> = runtimeState(initial = "Off") { it.talkModeStatusText }
266269
val talkModeConversation: StateFlow<List<VoiceConversationEntry>> =
267270
runtimeState(initial = emptyList()) { it.talkModeConversation }

‎apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1259,6 +1259,15 @@ class NodeRuntime private constructor(
12591259
val talkModeSpeaking: StateFlow<Boolean>
12601260
get() = talkMode.isSpeaking
12611261

1262+
val talkInputLevel: StateFlow<Float>
1263+
get() = talkMode.inputLevel
1264+
1265+
val talkOutputLevel: StateFlow<Float?>
1266+
get() = talkMode.outputLevel
1267+
1268+
val talkSpeechActive: StateFlow<Boolean>
1269+
get() = talkMode.speechActive
1270+
12621271
val talkModeStatusText: StateFlow<String>
12631272
get() = talkMode.statusText
12641273

‎apps/android/app/src/main/java/ai/openclaw/app/chat/VoiceNoteRecorderController.kt‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package ai.openclaw.app.chat
22

3+
import ai.openclaw.app.voice.smoothedAudioLevel
34
import android.content.Context
45
import android.media.MediaRecorder
56
import android.os.SystemClock
@@ -44,6 +45,9 @@ internal interface VoiceNoteRecordingEngine {
4445
fun stop(): Long
4546

4647
fun cancel()
48+
49+
/** Peak abs PCM amplitude (0..32767) since the last poll; 0 when not recording. */
50+
fun pollAmplitude(): Int
4751
}
4852

4953
/** Owns voice-note recording state and temporary-file cleanup. */
@@ -64,6 +68,9 @@ internal class VoiceNoteRecorderController(
6468
private val _elapsedMs = MutableStateFlow(0L)
6569
val elapsedMs: StateFlow<Long> = _elapsedMs.asStateFlow()
6670

71+
private val _inputLevel = MutableStateFlow(0f)
72+
val inputLevel: StateFlow<Float> = _inputLevel.asStateFlow()
73+
6774
private var outputFile: File? = null
6875
private var elapsedJob: Job? = null
6976
private var ownsMic = false
@@ -143,6 +150,7 @@ internal class VoiceNoteRecorderController(
143150
// coroutine is composition-scoped and may be cancelled before it runs,
144151
// so cancel() must still be able to delete the handed-off recording.
145152
_elapsedMs.value = 0L
153+
_inputLevel.value = 0f
146154
_state.value = VoiceNoteRecorderState.Preparing
147155
releaseMicLocked()
148156
VoiceNoteRecording(file = file, durationMs = durationMs)
@@ -171,6 +179,7 @@ internal class VoiceNoteRecorderController(
171179
outputFile?.delete()
172180
outputFile = null
173181
_elapsedMs.value = 0L
182+
_inputLevel.value = 0f
174183
_state.value = VoiceNoteRecorderState.Idle
175184
}
176185
}
@@ -190,13 +199,18 @@ internal class VoiceNoteRecorderController(
190199
while (isActive && state.value is VoiceNoteRecorderState.Recording) {
191200
val elapsed = (elapsedRealtimeMillis() - startedAt).coerceIn(0L, VOICE_NOTE_MAX_DURATION_MS)
192201
_elapsedMs.value = elapsed
202+
// MediaRecorder reports the peak since the last poll; at 10Hz that
203+
// reads close to the mean-abs mic levels used by the other waveform
204+
// surfaces, so peak/32767 needs no extra curve.
205+
val rawLevel = (engine.pollAmplitude().coerceIn(0, 32_767)) / 32_767f
206+
_inputLevel.value = smoothedAudioLevel(_inputLevel.value, rawLevel)
193207
// MediaRecorder's duration callback races its asynchronous auto-stop.
194208
// Own the cap here so every successful finish calls stop() exactly once.
195209
if (elapsed >= VOICE_NOTE_MAX_DURATION_MS) {
196210
finish()
197211
return@launch
198212
}
199-
delay(250L)
213+
delay(100L)
200214
}
201215
}
202216
}
@@ -213,6 +227,7 @@ internal class VoiceNoteRecorderController(
213227
releaseMicLocked()
214228
outputFile = null
215229
_elapsedMs.value = 0L
230+
_inputLevel.value = 0f
216231
_state.value = VoiceNoteRecorderState.Failure(message)
217232
}
218233

@@ -282,4 +297,8 @@ internal class AndroidVoiceNoteRecordingEngine(
282297
runCatching { active.stop() }
283298
active.release()
284299
}
300+
301+
// maxAmplitude throws until sampling starts on some OEMs; a dead meter beats
302+
// killing the recording.
303+
override fun pollAmplitude(): Int = recorder?.let { active -> runCatching { active.maxAmplitude }.getOrDefault(0) } ?: 0
285304
}

‎apps/android/app/src/main/java/ai/openclaw/app/ui/SettingsScreens.kt‎

Lines changed: 8 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,8 @@ import ai.openclaw.app.ui.design.ClawTextBadge
4949
import ai.openclaw.app.ui.design.ClawTextField
5050
import ai.openclaw.app.ui.design.ClawTheme
5151
import ai.openclaw.app.ui.design.OpenClawMascot
52+
import ai.openclaw.app.ui.design.TalkWaveform
53+
import ai.openclaw.app.ui.design.TalkWaveformPhase
5254
import android.Manifest
5355
import android.content.ClipData
5456
import android.content.ClipboardManager
@@ -628,16 +630,12 @@ private fun SettingsWaveformPanel(
628630
horizontalArrangement = Arrangement.spacedBy(5.dp),
629631
) {
630632
Icon(imageVector = Icons.Default.PlayArrow, contentDescription = null, modifier = Modifier.size(24.dp), tint = ClawTheme.colors.text)
631-
Row(modifier = Modifier.weight(1f), horizontalArrangement = Arrangement.SpaceEvenly, verticalAlignment = Alignment.CenterVertically) {
632-
listOf(6, 12, 18, 11, 28, 34, 18, 10, 8, 24, 38, 31, 12, 8, 18, 30, 40, 22, 12, 8, 20, 29, 16, 8).forEachIndexed { index, height ->
633-
Box(
634-
modifier =
635-
Modifier
636-
.size(width = 2.dp, height = (if (active) height else 7 + index % 4 * 4).dp)
637-
.background(if (active) ClawTheme.colors.text else ClawTheme.colors.textSubtle, RoundedCornerShape(999.dp)),
638-
)
639-
}
640-
}
633+
// Thinking is the preview phase: no capture runs on this screen, so the
634+
// synthetic swell demonstrates the animation without touching the mic.
635+
TalkWaveform(
636+
phase = if (active) TalkWaveformPhase.Thinking else TalkWaveformPhase.Idle,
637+
modifier = Modifier.weight(1f).height(48.dp),
638+
)
641639
}
642640
}
643641
}

0 commit comments

Comments
 (0)