Skip to content

Commit 816c700

Browse files
Merge remote-tracking branch 'origin/main' into fix/issue-30360
2 parents 019f784 + aee76a8 commit 816c700

86 files changed

Lines changed: 4582 additions & 460 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎apps/mobile/app.config.ts‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -367,7 +367,7 @@ const config: ExpoConfig = {
367367
"expo-audio",
368368
{
369369
microphonePermission: "Allow T3 Code to use your microphone for voice input.",
370-
recordAudioAndroid: false,
370+
recordAudioAndroid: true,
371371
enableBackgroundPlayback: false,
372372
enableBackgroundRecording: false,
373373
},
@@ -381,7 +381,14 @@ const config: ExpoConfig = {
381381
recordAudioAndroid: false,
382382
},
383383
],
384-
["expo-image-picker", { photosPermission: false, microphonePermission: false }],
384+
[
385+
"expo-image-picker",
386+
{
387+
photosPermission: false,
388+
// `false` makes this plugin add RECORD_AUDIO to blockedPermissions, breaking Android dictation.
389+
microphonePermission: "Allow T3 Code to use your microphone for voice input.",
390+
},
391+
],
385392
[
386393
"expo-splash-screen",
387394
{
Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
apply plugin: 'com.android.library'
2+
apply plugin: 'org.jetbrains.kotlin.android'
3+
4+
group = 'com.t3tools.voicetranscription'
5+
version = '0.0.0'
6+
7+
android {
8+
namespace 'expo.modules.t3voicetranscription'
9+
compileSdk rootProject.ext.compileSdkVersion
10+
11+
defaultConfig {
12+
minSdkVersion rootProject.ext.minSdkVersion
13+
targetSdkVersion rootProject.ext.targetSdkVersion
14+
}
15+
}
16+
17+
dependencies {
18+
implementation project(':expo-modules-core')
19+
implementation 'androidx.media3:media3-common:1.9.1'
20+
implementation 'com.google.mlkit:genai-speech-recognition:1.0.0-alpha2'
21+
testImplementation 'junit:junit:4.13.2'
22+
testImplementation 'org.robolectric:robolectric:4.16.1'
23+
testImplementation 'org.jetbrains.kotlinx:kotlinx-coroutines-test:1.10.2'
24+
}
25+
26+
tasks.withType(Test).configureEach {
27+
systemProperty 'robolectric.dependency.repo.url', 'https://repo.maven.apache.org/maven2'
28+
javaLauncher = javaToolchains.launcherFor {
29+
languageVersion = JavaLanguageVersion.of(21)
30+
}
31+
}
Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
2+
xmlns:tools="http://schemas.android.com/tools">
3+
<!-- Recognition is gated to API 31; older devices retain the rest of the app. -->
4+
<uses-sdk tools:overrideLibrary="com.google.mlkit.genai.speechrecognition,com.google.mlkit.genai.common" />
5+
</manifest>
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
1+
package expo.modules.t3voicetranscription
2+
3+
import android.content.Context
4+
import com.google.mlkit.genai.common.DownloadStatus
5+
import com.google.mlkit.genai.common.FeatureStatus
6+
import com.google.mlkit.genai.common.GenAiException
7+
import com.google.mlkit.genai.speechrecognition.SpeechRecognition
8+
import com.google.mlkit.genai.speechrecognition.SpeechRecognizerOptions
9+
import com.google.mlkit.genai.speechrecognition.SpeechRecognizerResponse
10+
import com.google.mlkit.genai.speechrecognition.speechRecognizerOptions
11+
import com.google.mlkit.genai.speechrecognition.speechRecognizerRequest
12+
import java.io.Closeable
13+
import java.util.Locale
14+
import kotlinx.coroutines.NonCancellable
15+
import kotlinx.coroutines.flow.Flow
16+
import kotlinx.coroutines.flow.first
17+
import kotlinx.coroutines.withContext
18+
19+
internal class VoiceTranscriptionException(
20+
val code: String,
21+
message: String,
22+
cause: Throwable? = null
23+
) : Exception(message, cause)
24+
25+
private val recognitionModes = listOf(
26+
SpeechRecognizerOptions.Mode.MODE_ADVANCED,
27+
SpeechRecognizerOptions.Mode.MODE_BASIC
28+
)
29+
30+
internal suspend fun isGoogleVoiceTranscriptionSupported(
31+
createClient: (Int) -> GoogleSpeechClient
32+
): Boolean {
33+
for (mode in recognitionModes) {
34+
createClient(mode).use { client ->
35+
when (checkModelSupport(client)) {
36+
FeatureStatus.AVAILABLE,
37+
FeatureStatus.DOWNLOADABLE,
38+
FeatureStatus.DOWNLOADING -> return true
39+
}
40+
}
41+
}
42+
return false
43+
}
44+
45+
internal suspend fun prepareGoogleVoiceTranscription(
46+
createClient: (Int) -> GoogleSpeechClient
47+
): Int {
48+
for (mode in recognitionModes) {
49+
createClient(mode).use { client ->
50+
if (prepareModel(client)) return mode
51+
}
52+
}
53+
throw VoiceTranscriptionException(
54+
"unavailable",
55+
"On-device voice transcription is not available for this device language."
56+
)
57+
}
58+
59+
private suspend fun prepareModel(client: GoogleSpeechClient): Boolean = try {
60+
val status = client.checkStatus()
61+
if (status == FeatureStatus.UNAVAILABLE) {
62+
false
63+
} else {
64+
if (status == FeatureStatus.DOWNLOADABLE ||
65+
status == FeatureStatus.DOWNLOADING
66+
) {
67+
client.download()
68+
}
69+
when (client.checkStatus()) {
70+
FeatureStatus.UNAVAILABLE -> false
71+
FeatureStatus.AVAILABLE -> true
72+
else -> throw VoiceTranscriptionException(
73+
"preparation-failed",
74+
"Voice transcription could not prepare this language."
75+
)
76+
}
77+
}
78+
} catch (error: GenAiException) {
79+
if (!isModelUnavailable(error)) throw error
80+
false
81+
}
82+
83+
// Any probe failure for one mode means "try the next mode"; prepare surfaces the real error.
84+
private suspend fun checkModelSupport(client: GoogleSpeechClient): Int = try {
85+
client.checkStatus()
86+
} catch (_: GenAiException) {
87+
FeatureStatus.UNAVAILABLE
88+
}
89+
90+
private fun isModelUnavailable(error: GenAiException): Boolean = when (error.errorCode) {
91+
GenAiException.ErrorCode.NOT_AVAILABLE,
92+
GenAiException.ErrorCode.NOT_SUPPORTED,
93+
GenAiException.ErrorCode.AICORE_INCOMPATIBLE,
94+
GenAiException.ErrorCode.NEEDS_SYSTEM_UPDATE -> true
95+
else -> false
96+
}
97+
98+
internal suspend fun collectGoogleVoiceTranscript(
99+
responses: Flow<SpeechRecognizerResponse>
100+
): String {
101+
val text = StringBuilder()
102+
var partial = ""
103+
var completed = false
104+
responses.collect { response ->
105+
when (response) {
106+
// Separators between final utterances are supplied by the SDK.
107+
is SpeechRecognizerResponse.FinalTextResponse -> {
108+
text.append(response.text)
109+
partial = ""
110+
}
111+
is SpeechRecognizerResponse.PartialTextResponse -> partial = response.text
112+
is SpeechRecognizerResponse.ErrorResponse -> throw response.e
113+
is SpeechRecognizerResponse.CompletedResponse -> completed = true
114+
else -> Unit
115+
}
116+
}
117+
requireCompleteAudio(completed) { "Speech recognition did not complete the recording." }
118+
// Some sessions complete with a trailing partial instead of another final event.
119+
return text.append(partial).toString().trim()
120+
}
121+
122+
internal interface GoogleSpeechClient : Closeable {
123+
suspend fun checkStatus(): Int
124+
suspend fun download()
125+
}
126+
127+
internal class MlKitSpeechClient(context: Context, locale: Locale, mode: Int) : GoogleSpeechClient {
128+
private val context = context.applicationContext
129+
private val recognizer = SpeechRecognition.getClient(
130+
speechRecognizerOptions {
131+
this.locale = locale
132+
preferredMode = mode
133+
}
134+
)
135+
136+
override suspend fun checkStatus() = recognizer.checkStatus()
137+
138+
override suspend fun download() {
139+
val result = recognizer.download().first {
140+
it is DownloadStatus.DownloadCompleted || it is DownloadStatus.DownloadFailed
141+
}
142+
if (result is DownloadStatus.DownloadFailed) throw result.e
143+
}
144+
145+
suspend fun transcribe(uri: String): String = withRecordedPcm(context, uri) { pcm ->
146+
PcmPipe.open().use { pipe ->
147+
val request = speechRecognizerRequest { audioSource = pipe.source }
148+
try {
149+
collectGoogleVoiceTranscript(
150+
pacedRecordedResponses(pcm, pipe) {
151+
recognizer.startRecognition(request)
152+
}
153+
)
154+
} finally {
155+
// Stop inference before closing the SDK-owned descriptor, including cancellation.
156+
withContext(NonCancellable) { runCatching { recognizer.stopRecognition() } }
157+
}
158+
}
159+
}
160+
161+
override fun close() = recognizer.close()
162+
}
Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
package expo.modules.t3voicetranscription
2+
3+
import com.google.mlkit.genai.speechrecognition.SpeechRecognizerResponse
4+
import kotlinx.coroutines.NonCancellable
5+
import kotlinx.coroutines.TimeoutCancellationException
6+
import kotlinx.coroutines.async
7+
import kotlinx.coroutines.coroutineScope
8+
import kotlinx.coroutines.delay
9+
import kotlinx.coroutines.flow.Flow
10+
import kotlinx.coroutines.flow.flow
11+
import kotlinx.coroutines.withContext
12+
import kotlinx.coroutines.withTimeout
13+
import java.io.File
14+
import java.util.concurrent.atomic.AtomicBoolean
15+
16+
internal interface PcmOutput : AutoCloseable {
17+
suspend fun write(buffer: ByteArray, count: Int)
18+
fun finish()
19+
fun verifyConsumed()
20+
}
21+
22+
private suspend fun replayPcm(file: File, output: PcmOutput) {
23+
val expected = file.length()
24+
var sent = 0L
25+
file.inputStream().use { input ->
26+
val buffer = ByteArray(3200)
27+
while (true) {
28+
val count = input.read(buffer)
29+
if (count == -1) break
30+
requireCompleteAudio(count > 0 && count % 2 == 0) {
31+
"Recorded PCM contains an incomplete audio frame."
32+
}
33+
// STREAMING requires real-time delivery, including the final short buffer.
34+
delay((count * 1000L + 31999) / 32000)
35+
output.write(buffer, count)
36+
sent += count
37+
}
38+
}
39+
requireCompleteAudio(sent == expected && file.length() == expected) {
40+
"The recording changed while speech recognition was reading it. No transcript was added."
41+
}
42+
}
43+
44+
internal fun pacedRecordedResponses(
45+
file: File,
46+
output: PcmOutput,
47+
responses: () -> Flow<SpeechRecognizerResponse>
48+
): Flow<SpeechRecognizerResponse> = flow {
49+
coroutineScope {
50+
val exhausted = AtomicBoolean()
51+
val sender = async {
52+
replayPcm(file, output)
53+
exhausted.set(true)
54+
output.finish()
55+
}
56+
try {
57+
withTimeout(file.length() / 32 + 30000) {
58+
responses().collect { emit(it) }
59+
requireCompleteAudio(exhausted.get()) {
60+
"Speech recognition ended before receiving the complete recording. " +
61+
"No transcript was added. Retry dictation."
62+
}
63+
sender.await()
64+
output.verifyConsumed()
65+
}
66+
} catch (e: TimeoutCancellationException) {
67+
throw VoiceTranscriptionException(
68+
"incomplete-audio",
69+
"Speech recognition did not finish the complete recording. No transcript was added. Retry dictation.",
70+
e
71+
)
72+
} finally {
73+
sender.cancel()
74+
try {
75+
output.finish()
76+
} finally {
77+
withContext(NonCancellable) { sender.join() }
78+
}
79+
}
80+
}
81+
}
82+
83+
internal inline fun requireCompleteAudio(condition: Boolean, message: () -> String) {
84+
if (!condition) throw VoiceTranscriptionException("incomplete-audio", message())
85+
}

0 commit comments

Comments
 (0)