Skip to content

Commit 7cfc66a

Browse files
fix(android): derive Voice readiness from Gateway catalog (#98269)
* fix(android): derive voice readiness from Gateway catalog Co-authored-by: Colin <[email protected]> * chore(android): sync native i18n inventory * test(gateway): use registered realtime provider ids * fix(gateway): satisfy Talk catalog lint * chore(android): refresh voice i18n inventory * fix(talk): preserve runtime readiness semantics * fix(talk): make catalog readiness authoritative * fix(talk): validate selected provider readiness * fix(android): honor authoritative talk readiness * fix(android): inventory voice readiness copy * fix(talk): report runtime-selected catalog provider --------- Co-authored-by: Peter Steinberger <[email protected]>
1 parent 4e6933c commit 7cfc66a

16 files changed

Lines changed: 1352 additions & 245 deletions

File tree

apps/.i18n/native-source.json

Lines changed: 295 additions & 183 deletions
Large diffs are not rendered by default.
Lines changed: 216 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,216 @@
1+
package ai.openclaw.app
2+
3+
import ai.openclaw.app.node.asObjectOrNull
4+
import ai.openclaw.app.node.asStringOrNull
5+
import kotlinx.serialization.json.JsonArray
6+
import kotlinx.serialization.json.JsonElement
7+
import kotlinx.serialization.json.JsonObject
8+
import kotlinx.serialization.json.JsonPrimitive
9+
import kotlinx.serialization.json.booleanOrNull
10+
11+
data class GatewayTalkSetupReadiness(
12+
val realtimeTalk: GatewayTalkSetupState,
13+
val dictation: GatewayTalkSetupState,
14+
) {
15+
companion object {
16+
fun unverified(
17+
issue: GatewayTalkSetupIssue = GatewayTalkSetupIssue.CatalogNotLoaded,
18+
): GatewayTalkSetupReadiness =
19+
GatewayTalkSetupReadiness(
20+
realtimeTalk = GatewayTalkSetupState.Unverified(issue),
21+
dictation = GatewayTalkSetupState.Unverified(issue),
22+
)
23+
}
24+
}
25+
26+
sealed interface GatewayTalkSetupState {
27+
data class Ready(
28+
val provider: GatewayTalkProvider,
29+
) : GatewayTalkSetupState
30+
31+
data class NeedsSetup(
32+
val issue: GatewayTalkSetupIssue,
33+
val provider: GatewayTalkProvider? = null,
34+
) : GatewayTalkSetupState
35+
36+
/** Catalog failures must not disable a startup path that the Gateway still validates. */
37+
data class Unverified(
38+
val issue: GatewayTalkSetupIssue,
39+
) : GatewayTalkSetupState
40+
}
41+
42+
enum class GatewayTalkSetupTarget(
43+
val title: String,
44+
) {
45+
REALTIME_TALK("Realtime Talk"),
46+
DICTATION("Dictation"),
47+
}
48+
49+
sealed interface GatewayTalkSetupIssue {
50+
data object CatalogNotLoaded : GatewayTalkSetupIssue
51+
52+
data object CatalogLoadFailed : GatewayTalkSetupIssue
53+
54+
data class GroupMissing(
55+
val target: GatewayTalkSetupTarget,
56+
) : GatewayTalkSetupIssue
57+
58+
data class NoProvider(
59+
val target: GatewayTalkSetupTarget,
60+
) : GatewayTalkSetupIssue
61+
62+
data class UnknownProvider(
63+
val target: GatewayTalkSetupTarget,
64+
val providerId: String,
65+
) : GatewayTalkSetupIssue
66+
67+
data class MissingReadiness(
68+
val target: GatewayTalkSetupTarget,
69+
) : GatewayTalkSetupIssue
70+
71+
data class ConfigureProvider(
72+
val target: GatewayTalkSetupTarget,
73+
) : GatewayTalkSetupIssue
74+
75+
data class MissingActiveProvider(
76+
val target: GatewayTalkSetupTarget,
77+
) : GatewayTalkSetupIssue
78+
79+
data class UnsupportedProvider(
80+
val target: GatewayTalkSetupTarget,
81+
) : GatewayTalkSetupIssue
82+
83+
data class ConfigureSelectedProvider(
84+
val providerLabel: String,
85+
) : GatewayTalkSetupIssue
86+
}
87+
88+
data class GatewayTalkProvider(
89+
val id: String,
90+
val label: String,
91+
)
92+
93+
val GatewayTalkSetupState.isReady: Boolean
94+
get() = this is GatewayTalkSetupState.Ready
95+
96+
val GatewayTalkSetupState.requiresSetup: Boolean
97+
get() = this is GatewayTalkSetupState.NeedsSetup
98+
99+
fun gatewayTalkSetupStatusText(state: GatewayTalkSetupState): String =
100+
when (state) {
101+
is GatewayTalkSetupState.Ready -> "Ready"
102+
is GatewayTalkSetupState.NeedsSetup -> "Needs setup"
103+
is GatewayTalkSetupState.Unverified -> "Unverified"
104+
}
105+
106+
fun gatewayTalkSetupDescription(state: GatewayTalkSetupState): String =
107+
when (state) {
108+
is GatewayTalkSetupState.Ready -> "${state.provider.label} via Gateway relay"
109+
is GatewayTalkSetupState.NeedsSetup -> gatewayTalkSetupIssueDescription(state.issue)
110+
is GatewayTalkSetupState.Unverified -> gatewayTalkSetupIssueDescription(state.issue)
111+
}
112+
113+
private fun gatewayTalkSetupIssueDescription(issue: GatewayTalkSetupIssue): String =
114+
when (issue) {
115+
GatewayTalkSetupIssue.CatalogNotLoaded -> "Gateway talk catalog not loaded"
116+
GatewayTalkSetupIssue.CatalogLoadFailed -> "Could not load Gateway talk catalog"
117+
is GatewayTalkSetupIssue.GroupMissing -> "Gateway did not return ${issue.target.title} setup"
118+
is GatewayTalkSetupIssue.NoProvider -> "No ${issue.target.title} provider is configured on the Gateway"
119+
is GatewayTalkSetupIssue.UnknownProvider -> "Gateway selected unknown provider ${issue.providerId}"
120+
is GatewayTalkSetupIssue.MissingReadiness -> "Gateway did not return ${issue.target.title} readiness"
121+
is GatewayTalkSetupIssue.ConfigureProvider -> "Configure a ${issue.target.title} provider on the Gateway"
122+
is GatewayTalkSetupIssue.MissingActiveProvider ->
123+
"Gateway did not identify the active ${issue.target.title} provider"
124+
is GatewayTalkSetupIssue.UnsupportedProvider ->
125+
"Choose a supported ${issue.target.title} provider on the Gateway"
126+
is GatewayTalkSetupIssue.ConfigureSelectedProvider -> "Configure ${issue.providerLabel} on the Gateway"
127+
}
128+
129+
internal fun parseGatewayTalkSetupReadiness(catalog: JsonObject?): GatewayTalkSetupReadiness {
130+
if (catalog == null) return GatewayTalkSetupReadiness.unverified()
131+
return GatewayTalkSetupReadiness(
132+
realtimeTalk =
133+
parseTalkCatalogGroup(catalog = catalog, key = "realtime", target = GatewayTalkSetupTarget.REALTIME_TALK),
134+
dictation =
135+
parseTalkCatalogGroup(catalog = catalog, key = "transcription", target = GatewayTalkSetupTarget.DICTATION),
136+
)
137+
}
138+
139+
private fun parseTalkCatalogGroup(
140+
catalog: JsonObject,
141+
key: String,
142+
target: GatewayTalkSetupTarget,
143+
): GatewayTalkSetupState {
144+
val group =
145+
catalog[key].asObjectOrNull()
146+
?: return GatewayTalkSetupState.Unverified(GatewayTalkSetupIssue.GroupMissing(target))
147+
val providers =
148+
(group["providers"] as? JsonArray)
149+
?.mapNotNull(::parseTalkCatalogProvider)
150+
.orEmpty()
151+
val ready = (group["ready"] as? JsonPrimitive)?.booleanOrNull
152+
val activeProviderId = group["activeProvider"].asStringOrNull()?.trim()?.takeIf(String::isNotEmpty)
153+
if (providers.isEmpty()) {
154+
return when {
155+
ready == false -> GatewayTalkSetupState.NeedsSetup(GatewayTalkSetupIssue.NoProvider(target))
156+
activeProviderId != null ->
157+
GatewayTalkSetupState.Unverified(GatewayTalkSetupIssue.UnknownProvider(target, activeProviderId))
158+
else -> GatewayTalkSetupState.Unverified(GatewayTalkSetupIssue.MissingReadiness(target))
159+
}
160+
}
161+
162+
if (activeProviderId == null) {
163+
if (ready == false) {
164+
return GatewayTalkSetupState.NeedsSetup(GatewayTalkSetupIssue.ConfigureProvider(target))
165+
}
166+
// Older Gateways can omit the selected provider and report alias-backed rows as unconfigured
167+
// even though session startup resolves them. Only an explicit readiness result is authoritative.
168+
return GatewayTalkSetupState.Unverified(GatewayTalkSetupIssue.MissingActiveProvider(target))
169+
}
170+
val selected =
171+
// Match Gateway registry precedence: canonical ids win before alias fallback.
172+
providers.firstOrNull { it.matchesId(activeProviderId) }
173+
?: providers.firstOrNull { it.matchesAlias(activeProviderId) }
174+
?: return if (ready == false) {
175+
GatewayTalkSetupState.NeedsSetup(GatewayTalkSetupIssue.UnsupportedProvider(target))
176+
} else {
177+
GatewayTalkSetupState.Unverified(GatewayTalkSetupIssue.UnknownProvider(target, activeProviderId))
178+
}
179+
val provider = GatewayTalkProvider(id = selected.id, label = selected.label)
180+
return when (ready) {
181+
true -> GatewayTalkSetupState.Ready(provider)
182+
false ->
183+
GatewayTalkSetupState.NeedsSetup(
184+
issue = GatewayTalkSetupIssue.ConfigureSelectedProvider(selected.label),
185+
provider = provider,
186+
)
187+
null -> GatewayTalkSetupState.Unverified(GatewayTalkSetupIssue.MissingReadiness(target))
188+
}
189+
}
190+
191+
private data class TalkCatalogProvider(
192+
val id: String,
193+
val label: String,
194+
val configured: Boolean,
195+
val aliases: List<String>,
196+
) {
197+
fun matchesId(candidate: String): Boolean = id.equals(candidate, ignoreCase = true)
198+
199+
fun matchesAlias(candidate: String): Boolean = aliases.any { it.equals(candidate, ignoreCase = true) }
200+
}
201+
202+
private fun parseTalkCatalogProvider(item: JsonElement): TalkCatalogProvider? {
203+
val value = item.asObjectOrNull() ?: return null
204+
val id = value["id"].asStringOrNull()?.trim()?.takeIf(String::isNotEmpty) ?: return null
205+
val label = value["label"].asStringOrNull()?.trim()?.takeIf(String::isNotEmpty) ?: id
206+
val aliases =
207+
(value["aliases"] as? JsonArray)
208+
?.mapNotNull { it.asStringOrNull()?.trim()?.takeIf(String::isNotEmpty) }
209+
.orEmpty()
210+
return TalkCatalogProvider(
211+
id = id,
212+
label = label,
213+
configured = (value["configured"] as? JsonPrimitive)?.booleanOrNull == true,
214+
aliases = aliases,
215+
)
216+
}

apps/android/app/src/main/java/ai/openclaw/app/MainViewModel.kt

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,8 @@ class MainViewModel(
127127
val modelAuthProviders: StateFlow<List<GatewayModelProviderSummary>> = runtimeState(initial = emptyList()) { it.modelAuthProviders }
128128
val modelCatalogRefreshing: StateFlow<Boolean> = runtimeState(initial = false) { it.modelCatalogRefreshing }
129129
val modelCatalogErrorText: StateFlow<String?> = runtimeState(initial = null) { it.modelCatalogErrorText }
130+
val talkSetupReadiness: StateFlow<GatewayTalkSetupReadiness> =
131+
runtimeState(initial = GatewayTalkSetupReadiness.unverified()) { it.talkSetupReadiness }
130132
val gatewayDefaultAgentId: StateFlow<String?> = runtimeState(initial = null) { it.gatewayDefaultAgentId }
131133
val gatewayAgents: StateFlow<List<GatewayAgentSummary>> = runtimeState(initial = emptyList()) { it.gatewayAgents }
132134
val cronStatus: StateFlow<GatewayCronStatus> = runtimeState(initial = GatewayCronStatus(enabled = false, jobs = 0, nextWakeAtMs = null)) { it.cronStatus }
@@ -527,6 +529,10 @@ class MainViewModel(
527529
ensureRuntime().refreshModelCatalog()
528530
}
529531

532+
fun refreshTalkSetupReadiness() {
533+
ensureRuntime().refreshTalkSetupReadiness()
534+
}
535+
530536
fun refreshAgents() {
531537
ensureRuntime().refreshAgents()
532538
}

apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -549,6 +549,8 @@ class NodeRuntime(
549549
val modelCatalogRefreshing: StateFlow<Boolean> = _modelCatalogRefreshing.asStateFlow()
550550
private val _modelCatalogErrorText = MutableStateFlow<String?>(null)
551551
val modelCatalogErrorText: StateFlow<String?> = _modelCatalogErrorText.asStateFlow()
552+
private val _talkSetupReadiness = MutableStateFlow(GatewayTalkSetupReadiness.unverified())
553+
val talkSetupReadiness: StateFlow<GatewayTalkSetupReadiness> = _talkSetupReadiness.asStateFlow()
552554
private val _gatewayDefaultAgentId = MutableStateFlow<String?>(null)
553555
val gatewayDefaultAgentId: StateFlow<String?> = _gatewayDefaultAgentId.asStateFlow()
554556
private val _gatewayAgents = MutableStateFlow<List<GatewayAgentSummary>>(emptyList())
@@ -667,6 +669,7 @@ class NodeRuntime(
667669
_gatewayAgents.value = emptyList()
668670
_modelCatalog.value = emptyList()
669671
_modelAuthProviders.value = emptyList()
672+
_talkSetupReadiness.value = GatewayTalkSetupReadiness.unverified()
670673
_cronStatus.value = GatewayCronStatus(enabled = false, jobs = 0, nextWakeAtMs = null)
671674
_cronJobs.value = emptyList()
672675
_usageSummary.value = GatewayUsageSummary(updatedAtMs = null, providers = emptyList())
@@ -1052,6 +1055,7 @@ class NodeRuntime(
10521055
refreshBrandingFromGateway()
10531056
refreshAgentsFromGateway()
10541057
refreshModelCatalogFromGateway()
1058+
refreshTalkSetupReadinessFromGateway()
10551059
refreshCronFromGateway()
10561060
refreshUsageFromGateway()
10571061
refreshSkillsFromGateway()
@@ -1068,6 +1072,10 @@ class NodeRuntime(
10681072
}
10691073
}
10701074

1075+
fun refreshTalkSetupReadiness() {
1076+
scope.launch { refreshTalkSetupReadinessFromGateway() }
1077+
}
1078+
10711079
fun refreshAgents() {
10721080
scope.launch {
10731081
refreshAgentsFromGateway()
@@ -1501,6 +1509,8 @@ class NodeRuntime(
15011509
fun setVoiceScreenActive(active: Boolean) {
15021510
if (!active) {
15031511
stopManualVoiceSession()
1512+
} else {
1513+
refreshTalkSetupReadiness()
15041514
}
15051515
// Don't re-enable on active=true; mic toggle drives that
15061516
}
@@ -2322,6 +2332,20 @@ class NodeRuntime(
23222332
}
23232333
}
23242334

2335+
private suspend fun refreshTalkSetupReadinessFromGateway() {
2336+
if (!operatorConnected) {
2337+
_talkSetupReadiness.value = GatewayTalkSetupReadiness.unverified()
2338+
return
2339+
}
2340+
_talkSetupReadiness.value =
2341+
try {
2342+
val response = operatorSession.request("talk.catalog", "{}")
2343+
parseGatewayTalkSetupReadiness(json.parseToJsonElement(response).asObjectOrNull())
2344+
} catch (_: Throwable) {
2345+
GatewayTalkSetupReadiness.unverified(GatewayTalkSetupIssue.CatalogLoadFailed)
2346+
}
2347+
}
2348+
23252349
private suspend fun refreshCronFromGateway() {
23262350
_cronRefreshing.value = true
23272351
_cronErrorText.value = null

apps/android/app/src/main/java/ai/openclaw/app/ui/SettingsScreens.kt

Lines changed: 30 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -8,13 +8,18 @@ import ai.openclaw.app.GatewayConnectionDisplay
88
import ai.openclaw.app.GatewayConnectionProblem
99
import ai.openclaw.app.GatewayCronJobSummary
1010
import ai.openclaw.app.GatewayExecApprovalSummary
11+
import ai.openclaw.app.GatewayTalkSetupReadiness
12+
import ai.openclaw.app.GatewayTalkSetupState
1113
import ai.openclaw.app.GatewayUsageProviderSummary
1214
import ai.openclaw.app.LocationMode
1315
import ai.openclaw.app.MainViewModel
1416
import ai.openclaw.app.NotificationPackageFilterMode
1517
import ai.openclaw.app.SensitiveFeatureConfig
1618
import ai.openclaw.app.chat.ChatPendingToolCall
19+
import ai.openclaw.app.gatewayTalkSetupDescription
20+
import ai.openclaw.app.gatewayTalkSetupStatusText
1721
import ai.openclaw.app.hasPhotoReadPermission
22+
import ai.openclaw.app.isReady
1823
import ai.openclaw.app.loadAndroidLicenseNotices
1924
import ai.openclaw.app.node.DeviceNotificationListenerService
2025
import ai.openclaw.app.photoReadPermissionsForRequest
@@ -407,14 +412,16 @@ private fun VoiceSettingsScreen(
407412
onBack: () -> Unit,
408413
) {
409414
val speakerEnabled by viewModel.speakerEnabled.collectAsState()
410-
val micEnabled by viewModel.micEnabled.collectAsState()
411-
val talkModeEnabled by viewModel.talkModeEnabled.collectAsState()
415+
val isConnected by viewModel.isConnected.collectAsState()
416+
val talkSetupReadiness by viewModel.talkSetupReadiness.collectAsState()
417+
418+
LaunchedEffect(isConnected) {
419+
if (isConnected) viewModel.refreshTalkSetupReadiness()
420+
}
412421

413422
SettingsDetailFrame(title = "Talk Provider Setup", subtitle = "Configure voice, transport, and playback.", icon = Icons.Default.Mic, onBack = onBack) {
414423
Column(verticalArrangement = Arrangement.spacedBy(10.dp)) {
415-
VoiceSetupPanel(
416-
voiceActive = micEnabled || talkModeEnabled,
417-
)
424+
VoiceSetupPanel(talkSetupReadiness)
418425
Text(text = "Audio Test", style = ClawTheme.type.section, color = ClawTheme.colors.text)
419426
Text(text = "Check that OpenClaw can speak clearly on this phone.", style = ClawTheme.type.body, color = ClawTheme.colors.textMuted)
420427
SettingsWaveformPanel(active = speakerEnabled, onClick = ::playVoiceSetupTone)
@@ -433,33 +440,29 @@ private fun VoiceSettingsScreen(
433440

434441
@Composable
435442
private fun VoiceSetupPanel(
436-
voiceActive: Boolean,
443+
readiness: GatewayTalkSetupReadiness,
437444
) {
438445
Column(verticalArrangement = Arrangement.spacedBy(9.dp)) {
439-
VoiceSetupActionRow(
440-
title = "Realtime Provider",
441-
subtitle = "Gateway voice relay",
442-
icon = Icons.Default.GraphicEq,
443-
statusText = if (voiceActive) "Live" else "Ready",
444-
ready = true,
445-
)
446-
VoiceSetupActionRow(
447-
title = "Voice",
448-
subtitle = "Voice input",
449-
icon = Icons.Default.Mic,
450-
statusText = "Configured",
451-
ready = true,
452-
)
453-
VoiceSetupActionRow(
454-
title = "Transport",
455-
subtitle = "Socket relay",
456-
icon = Icons.Default.Bolt,
457-
statusText = "Configured",
458-
ready = true,
459-
)
446+
VoiceSetupReadinessRow(title = "Realtime Talk", state = readiness.realtimeTalk, icon = Icons.Default.GraphicEq)
447+
VoiceSetupReadinessRow(title = "Dictation", state = readiness.dictation, icon = Icons.Default.Mic)
460448
}
461449
}
462450

451+
@Composable
452+
private fun VoiceSetupReadinessRow(
453+
title: String,
454+
state: GatewayTalkSetupState,
455+
icon: ImageVector,
456+
) {
457+
VoiceSetupActionRow(
458+
title = title,
459+
subtitle = gatewayTalkSetupDescription(state),
460+
icon = icon,
461+
statusText = gatewayTalkSetupStatusText(state),
462+
ready = state.isReady,
463+
)
464+
}
465+
463466
@Composable
464467
private fun VoiceSetupActionRow(
465468
title: String,

0 commit comments

Comments
 (0)