Skip to content

Commit 339f6d5

Browse files
Merge branch 'main' into agents/Vladimir-0001/preview-screencast-timeout
2 parents b60a7bd + b46abbf commit 339f6d5

509 files changed

Lines changed: 13926 additions & 6906 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎apps/mobile/modules/t3-voice-transcription/android/build.gradle‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,11 @@ android {
1616

1717
dependencies {
1818
implementation project(':expo-modules-core')
19+
implementation project(':expo-audio')
1920
implementation 'androidx.media3:media3-common:1.9.1'
2021
implementation 'com.google.mlkit:genai-speech-recognition:1.0.0-alpha2'
2122
testImplementation 'junit:junit:4.13.2'
23+
testImplementation 'com.facebook.react:react-android'
2224
testImplementation 'org.robolectric:robolectric:4.16.1'
2325
testImplementation 'org.jetbrains.kotlinx:kotlinx-coroutines-test:1.10.2'
2426
}

‎apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/GoogleVoiceTranscriber.kt‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -158,5 +158,16 @@ internal class MlKitSpeechClient(context: Context, locale: Locale, mode: Int) :
158158
}
159159
}
160160

161+
suspend fun transcribeLive(capture: LivePcmCapture, pipe: PcmPipe): String = pipe.use {
162+
val request = speechRecognizerRequest { audioSource = pipe.source }
163+
try {
164+
collectGoogleVoiceTranscript(
165+
livePcmResponses(capture, pipe) { recognizer.startRecognition(request) }
166+
)
167+
} finally {
168+
withContext(NonCancellable) { runCatching { recognizer.stopRecognition() } }
169+
}
170+
}
171+
161172
override fun close() = recognizer.close()
162173
}
Lines changed: 216 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,216 @@
1+
package expo.modules.t3voicetranscription
2+
3+
import java.io.File
4+
import kotlin.math.abs
5+
import kotlin.math.log10
6+
import kotlin.time.TimeSource
7+
import kotlinx.coroutines.CancellationException
8+
import kotlinx.coroutines.CompletableDeferred
9+
import kotlinx.coroutines.CoroutineScope
10+
import kotlinx.coroutines.Job
11+
import kotlinx.coroutines.NonCancellable
12+
import kotlinx.coroutines.TimeoutCancellationException
13+
import kotlinx.coroutines.currentCoroutineContext
14+
import kotlinx.coroutines.coroutineScope
15+
import kotlinx.coroutines.channels.Channel
16+
import kotlinx.coroutines.delay
17+
import kotlinx.coroutines.ensureActive
18+
import kotlinx.coroutines.launch
19+
import kotlinx.coroutines.withContext
20+
import kotlinx.coroutines.withTimeout
21+
22+
internal interface PcmCaptureSource {
23+
val capacityBytes: Int
24+
fun start()
25+
suspend fun read(buffer: ByteArray, blocking: Boolean): Int
26+
fun stop()
27+
fun release()
28+
}
29+
30+
internal class LivePcmCapture(
31+
private val source: PcmCaptureSource,
32+
val file: File,
33+
private val output: PcmOutput,
34+
private val clock: TimeSource = TimeSource.Monotonic
35+
) {
36+
@Volatile private var worker: Job? = null
37+
private val completion = CompletableDeferred<Result<Unit>>()
38+
private val microphoneStopped = CompletableDeferred<Result<Unit>>()
39+
40+
@Volatile private var stopping = false
41+
private var sourceStopped = false
42+
private var sourceReleased = false
43+
44+
@Volatile var bytes = 0L
45+
private set
46+
47+
@Volatile var metering = -160.0
48+
private set
49+
val isFinished get() = completion.isCompleted
50+
51+
@Volatile var inputExhausted = false
52+
private set
53+
val hasStarted get() = worker != null
54+
val isRecording get() = hasStarted && !microphoneStopped.isCompleted
55+
56+
fun start(scope: CoroutineScope, seconds: Double, onFinished: (Throwable?) -> Unit) {
57+
check(worker == null && !sourceReleased) { "The microphone is not prepared." }
58+
worker = scope.launch {
59+
var result = runCatching {
60+
source.start()
61+
writeCapture(seconds, onFinished)
62+
requireCompleteAudio(bytes > 0 && file.length() == bytes) {
63+
"The microphone did not finish its captured audio."
64+
}
65+
}
66+
result = finishSource(result)
67+
inputExhausted = result.isSuccess
68+
result = runCatching { output.finish() }.fold(
69+
onSuccess = { result },
70+
onFailure = { failure -> mergeFailure(result, failure) }
71+
)
72+
if (microphoneStopped.complete(result)) onFinished(result.exceptionOrNull())
73+
completion.complete(result)
74+
}.also { job ->
75+
job.invokeOnCompletion { error ->
76+
if (!completion.isCompleted) {
77+
val result =
78+
finishSource(Result.failure(error ?: CancellationException("Capture stopped.")))
79+
completion.complete(result)
80+
microphoneStopped.complete(result)
81+
runCatching { output.finish() }
82+
}
83+
}
84+
}
85+
}
86+
87+
private suspend fun writeCapture(
88+
seconds: Double,
89+
onFinished: (Throwable?) -> Unit
90+
) = coroutineScope {
91+
val started = clock.markNow()
92+
val chunks = Channel<ByteArray>(100)
93+
var sent = 0L
94+
val sender = launch {
95+
var deadline = 0L
96+
for (chunk in chunks) {
97+
deadline = maxOf(
98+
deadline + (chunk.size * 1000L + 31999) / 32000,
99+
started.elapsedNow().inWholeMilliseconds
100+
)
101+
delay((deadline - started.elapsedNow().inWholeMilliseconds).coerceAtLeast(0))
102+
output.write(chunk, chunk.size)
103+
deadline = started.elapsedNow().inWholeMilliseconds
104+
sent += chunk.size
105+
}
106+
}
107+
val result = finishSource(runCatching { captureChunks(seconds, chunks) })
108+
microphoneStopped.complete(result)
109+
if (result.exceptionOrNull() !is CancellationException) onFinished(result.exceptionOrNull())
110+
chunks.close()
111+
result.getOrThrow()
112+
sender.join()
113+
requireCompleteAudio(sent == bytes) { "Speech recognition did not receive all captured audio." }
114+
}
115+
116+
private suspend fun captureChunks(seconds: Double, chunks: Channel<ByteArray>) {
117+
val buffer = ByteArray(3200)
118+
var drained = 0
119+
file.outputStream().use { stream ->
120+
while (true) {
121+
currentCoroutineContext().ensureActive()
122+
val draining = stopping || bytes >= seconds * 32000
123+
val count = source.read(buffer, !draining)
124+
requireCompleteAudio(count in 0..buffer.size && count % 2 == 0) {
125+
"The microphone returned incomplete audio."
126+
}
127+
if (count == 0) {
128+
requireCompleteAudio(draining) { "The microphone stopped supplying audio." }
129+
break
130+
}
131+
if (draining) {
132+
drained += count
133+
requireCompleteAudio(drained <= source.capacityBytes + buffer.size) {
134+
"The microphone audio queue did not drain."
135+
}
136+
}
137+
stream.write(buffer, 0, count)
138+
// Keep draining the microphone while inference starts or applies backpressure.
139+
if (!chunks.trySend(buffer.copyOf(count)).isSuccess) {
140+
throw VoiceTranscriptionException(
141+
"audio-overflow",
142+
"Speech recognition fell behind capture."
143+
)
144+
}
145+
bytes += count
146+
var peak = 0
147+
for (i in 0 until count step 2) {
148+
val sample = ((buffer[i].toInt() and 255) or (buffer[i + 1].toInt() shl 8)).toShort()
149+
peak = maxOf(peak, abs(sample.toInt()))
150+
}
151+
metering = if (peak == 0) -160.0 else 20 * log10(peak / 32768.0)
152+
}
153+
}
154+
}
155+
156+
suspend fun stop() {
157+
stopping = true
158+
if (worker == null) {
159+
finishSource(Result.success(Unit)).getOrThrow()
160+
output.finish()
161+
return
162+
}
163+
try {
164+
withTimeout(2000) { microphoneStopped.await().getOrThrow() }
165+
} catch (error: TimeoutCancellationException) {
166+
cancel()
167+
throw VoiceTranscriptionException("incomplete-audio", "The microphone did not stop.", error)
168+
}
169+
}
170+
171+
suspend fun awaitFinished() = completion.await().getOrThrow()
172+
173+
fun abort() {
174+
stopping = true
175+
worker?.cancel()
176+
runCatching { stopSource() }
177+
runCatching { output.finish() }
178+
if (worker == null) {
179+
val result = finishSource(Result.failure(CancellationException("Capture was cancelled.")))
180+
completion.complete(result)
181+
microphoneStopped.complete(result)
182+
}
183+
}
184+
185+
suspend fun cancel() = withContext(NonCancellable) {
186+
abort()
187+
worker?.join()
188+
finishSource(Result.success(Unit))
189+
}
190+
191+
@Synchronized private fun stopSource() {
192+
if (sourceStopped) return
193+
sourceStopped = true
194+
source.stop()
195+
}
196+
197+
@Synchronized private fun finishSource(result: Result<Unit>): Result<Unit> {
198+
if (sourceReleased) return result
199+
sourceReleased = true
200+
var outcome = runCatching { stopSource() }.fold(
201+
onSuccess = { result },
202+
onFailure = { mergeFailure(result, it) }
203+
)
204+
outcome = runCatching { source.release() }.fold(
205+
onSuccess = { outcome },
206+
onFailure = { mergeFailure(outcome, it) }
207+
)
208+
return outcome
209+
}
210+
}
211+
212+
private fun mergeFailure(result: Result<Unit>, failure: Throwable): Result<Unit> {
213+
val original = result.exceptionOrNull() ?: return Result.failure(failure)
214+
if (original !== failure) original.addSuppressed(failure)
215+
return result
216+
}

0 commit comments

Comments
 (0)