Skip to content

Commit 4868826

Browse files
committed
fix(mobile): keep every Android dictation utterance
1 parent f00a386 commit 4868826

2 files changed

Lines changed: 52 additions & 9 deletions

File tree

‎apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/GoogleVoiceTranscriber.kt‎

Lines changed: 23 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import com.google.mlkit.genai.speechrecognition.speechRecognizerRequest
1414
import java.io.Closeable
1515
import java.util.Locale
1616
import kotlinx.coroutines.NonCancellable
17+
import kotlinx.coroutines.flow.Flow
1718
import kotlinx.coroutines.flow.first
1819
import kotlinx.coroutines.withContext
1920

@@ -63,6 +64,27 @@ private suspend fun checkModelSupport(client: GoogleSpeechClient): Int = try {
6364
FeatureStatus.UNAVAILABLE
6465
}
6566

67+
internal suspend fun collectGoogleVoiceTranscript(
68+
responses: Flow<SpeechRecognizerResponse>
69+
): String {
70+
val text = StringBuilder()
71+
var partial = ""
72+
responses.collect { response ->
73+
when (response) {
74+
// ONE_SHOT files can emit multiple final utterances, with separators supplied by the SDK.
75+
is SpeechRecognizerResponse.FinalTextResponse -> {
76+
text.append(response.text)
77+
partial = ""
78+
}
79+
is SpeechRecognizerResponse.PartialTextResponse -> partial = response.text
80+
is SpeechRecognizerResponse.ErrorResponse -> throw response.e
81+
else -> Unit
82+
}
83+
}
84+
// Some file sessions complete with a trailing partial instead of another final event.
85+
return text.append(partial).toString().trim()
86+
}
87+
6688
internal class MlKitSpeechClient(context: Context, locale: Locale, mode: Int) : GoogleSpeechClient {
6789
private val context = context.applicationContext
6890
private val recognizer = SpeechRecognition.getClient(
@@ -86,16 +108,8 @@ internal class MlKitSpeechClient(context: Context, locale: Locale, mode: Int) :
86108
Uri.parse(uri)
87109
).use { source ->
88110
val request = speechRecognizerRequest { audioSource = source }
89-
var text = ""
90111
try {
91-
recognizer.startRecognition(request).collect { response ->
92-
when (response) {
93-
is SpeechRecognizerResponse.FinalTextResponse -> text = response.text
94-
is SpeechRecognizerResponse.ErrorResponse -> throw response.e
95-
else -> Unit
96-
}
97-
}
98-
text.trim()
112+
collectGoogleVoiceTranscript(recognizer.startRecognition(request))
99113
} finally {
100114
// Stop inference before closing its audio descriptor, including coroutine cancellation.
101115
withContext(NonCancellable) { runCatching { recognizer.stopRecognition() } }

‎apps/mobile/modules/t3-voice-transcription/android/src/test/java/expo/modules/t3voicetranscription/GoogleVoiceTranscriberTest.kt‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,44 @@ package expo.modules.t3voicetranscription
33
import com.google.mlkit.genai.common.FeatureStatus
44
import com.google.mlkit.genai.common.GenAiException
55
import com.google.mlkit.genai.speechrecognition.SpeechRecognizerOptions
6+
import com.google.mlkit.genai.speechrecognition.SpeechRecognizerResponse
67
import kotlinx.coroutines.CompletableDeferred
78
import kotlinx.coroutines.cancelAndJoin
9+
import kotlinx.coroutines.flow.flow
810
import kotlinx.coroutines.launch
911
import kotlinx.coroutines.runBlocking
1012
import org.junit.Assert.assertEquals
1113
import org.junit.Assert.assertTrue
1214
import org.junit.Test
1315

1416
class GoogleVoiceTranscriberTest {
17+
@Test
18+
fun preservesUtterancesAndTheirSpacingWithoutDuplicatingHypotheses() = runBlocking {
19+
for ((finals, trailing, expected) in listOf(
20+
Triple(
21+
listOf("The beginning.", " The middle.", " Repeat this.", " Repeat this."),
22+
emptyList(),
23+
"The beginning. The middle. Repeat this. Repeat this."
24+
),
25+
Triple(listOf("最初の文。", "次の文。"), emptyList(), "最初の文。次の文。"),
26+
Triple(
27+
listOf("The beginning."),
28+
listOf(" An unfinished ending", " The final sentence."),
29+
"The beginning. The final sentence."
30+
)
31+
)) {
32+
val responses = flow {
33+
for (text in finals) {
34+
emit(SpeechRecognizerResponse.PartialTextResponse("Unfinished hypothesis"))
35+
emit(SpeechRecognizerResponse.FinalTextResponse(text, emptyList()))
36+
}
37+
for (text in trailing) emit(SpeechRecognizerResponse.PartialTextResponse(text))
38+
emit(SpeechRecognizerResponse.CompletedResponse)
39+
}
40+
assertEquals(expected, collectGoogleVoiceTranscript(responses))
41+
}
42+
}
43+
1544
private class Client(
1645
var status: Int,
1746
val onStatus: () -> Unit = {},

0 commit comments

Comments
 (0)