|
| 1 | +package expo.modules.t3voicetranscription |
| 2 | + |
| 3 | +import java.io.File |
| 4 | +import kotlin.math.abs |
| 5 | +import kotlin.math.log10 |
| 6 | +import kotlin.time.TimeSource |
| 7 | +import kotlinx.coroutines.CancellationException |
| 8 | +import kotlinx.coroutines.CompletableDeferred |
| 9 | +import kotlinx.coroutines.CoroutineScope |
| 10 | +import kotlinx.coroutines.Job |
| 11 | +import kotlinx.coroutines.NonCancellable |
| 12 | +import kotlinx.coroutines.TimeoutCancellationException |
| 13 | +import kotlinx.coroutines.currentCoroutineContext |
| 14 | +import kotlinx.coroutines.coroutineScope |
| 15 | +import kotlinx.coroutines.channels.Channel |
| 16 | +import kotlinx.coroutines.delay |
| 17 | +import kotlinx.coroutines.ensureActive |
| 18 | +import kotlinx.coroutines.launch |
| 19 | +import kotlinx.coroutines.withContext |
| 20 | +import kotlinx.coroutines.withTimeout |
| 21 | + |
| 22 | +internal interface PcmCaptureSource { |
| 23 | + val capacityBytes: Int |
| 24 | + fun start() |
| 25 | + suspend fun read(buffer: ByteArray, blocking: Boolean): Int |
| 26 | + fun stop() |
| 27 | + fun release() |
| 28 | +} |
| 29 | + |
| 30 | +internal class LivePcmCapture( |
| 31 | + private val source: PcmCaptureSource, |
| 32 | + val file: File, |
| 33 | + private val output: PcmOutput, |
| 34 | + private val clock: TimeSource = TimeSource.Monotonic |
| 35 | +) { |
| 36 | + @Volatile private var worker: Job? = null |
| 37 | + private val completion = CompletableDeferred<Result<Unit>>() |
| 38 | + private val microphoneStopped = CompletableDeferred<Result<Unit>>() |
| 39 | + |
| 40 | + @Volatile private var stopping = false |
| 41 | + private var sourceStopped = false |
| 42 | + private var sourceReleased = false |
| 43 | + |
| 44 | + @Volatile var bytes = 0L |
| 45 | + private set |
| 46 | + |
| 47 | + @Volatile var metering = -160.0 |
| 48 | + private set |
| 49 | + val isFinished get() = completion.isCompleted |
| 50 | + |
| 51 | + @Volatile var inputExhausted = false |
| 52 | + private set |
| 53 | + val hasStarted get() = worker != null |
| 54 | + val isRecording get() = hasStarted && !microphoneStopped.isCompleted |
| 55 | + |
| 56 | + fun start(scope: CoroutineScope, seconds: Double, onFinished: (Throwable?) -> Unit) { |
| 57 | + check(worker == null && !sourceReleased) { "The microphone is not prepared." } |
| 58 | + worker = scope.launch { |
| 59 | + var result = runCatching { |
| 60 | + source.start() |
| 61 | + writeCapture(seconds, onFinished) |
| 62 | + requireCompleteAudio(bytes > 0 && file.length() == bytes) { |
| 63 | + "The microphone did not finish its captured audio." |
| 64 | + } |
| 65 | + } |
| 66 | + result = finishSource(result) |
| 67 | + inputExhausted = result.isSuccess |
| 68 | + result = runCatching { output.finish() }.fold( |
| 69 | + onSuccess = { result }, |
| 70 | + onFailure = { failure -> mergeFailure(result, failure) } |
| 71 | + ) |
| 72 | + if (microphoneStopped.complete(result)) onFinished(result.exceptionOrNull()) |
| 73 | + completion.complete(result) |
| 74 | + }.also { job -> |
| 75 | + job.invokeOnCompletion { error -> |
| 76 | + if (!completion.isCompleted) { |
| 77 | + val result = |
| 78 | + finishSource(Result.failure(error ?: CancellationException("Capture stopped."))) |
| 79 | + completion.complete(result) |
| 80 | + microphoneStopped.complete(result) |
| 81 | + runCatching { output.finish() } |
| 82 | + } |
| 83 | + } |
| 84 | + } |
| 85 | + } |
| 86 | + |
| 87 | + private suspend fun writeCapture( |
| 88 | + seconds: Double, |
| 89 | + onFinished: (Throwable?) -> Unit |
| 90 | + ) = coroutineScope { |
| 91 | + val started = clock.markNow() |
| 92 | + val chunks = Channel<ByteArray>(100) |
| 93 | + var sent = 0L |
| 94 | + val sender = launch { |
| 95 | + var deadline = 0L |
| 96 | + for (chunk in chunks) { |
| 97 | + deadline = maxOf( |
| 98 | + deadline + (chunk.size * 1000L + 31999) / 32000, |
| 99 | + started.elapsedNow().inWholeMilliseconds |
| 100 | + ) |
| 101 | + delay((deadline - started.elapsedNow().inWholeMilliseconds).coerceAtLeast(0)) |
| 102 | + output.write(chunk, chunk.size) |
| 103 | + deadline = started.elapsedNow().inWholeMilliseconds |
| 104 | + sent += chunk.size |
| 105 | + } |
| 106 | + } |
| 107 | + val result = finishSource(runCatching { captureChunks(seconds, chunks) }) |
| 108 | + microphoneStopped.complete(result) |
| 109 | + if (result.exceptionOrNull() !is CancellationException) onFinished(result.exceptionOrNull()) |
| 110 | + chunks.close() |
| 111 | + result.getOrThrow() |
| 112 | + sender.join() |
| 113 | + requireCompleteAudio(sent == bytes) { "Speech recognition did not receive all captured audio." } |
| 114 | + } |
| 115 | + |
| 116 | + private suspend fun captureChunks(seconds: Double, chunks: Channel<ByteArray>) { |
| 117 | + val buffer = ByteArray(3200) |
| 118 | + var drained = 0 |
| 119 | + file.outputStream().use { stream -> |
| 120 | + while (true) { |
| 121 | + currentCoroutineContext().ensureActive() |
| 122 | + val draining = stopping || bytes >= seconds * 32000 |
| 123 | + val count = source.read(buffer, !draining) |
| 124 | + requireCompleteAudio(count in 0..buffer.size && count % 2 == 0) { |
| 125 | + "The microphone returned incomplete audio." |
| 126 | + } |
| 127 | + if (count == 0) { |
| 128 | + requireCompleteAudio(draining) { "The microphone stopped supplying audio." } |
| 129 | + break |
| 130 | + } |
| 131 | + if (draining) { |
| 132 | + drained += count |
| 133 | + requireCompleteAudio(drained <= source.capacityBytes + buffer.size) { |
| 134 | + "The microphone audio queue did not drain." |
| 135 | + } |
| 136 | + } |
| 137 | + stream.write(buffer, 0, count) |
| 138 | + // Keep draining the microphone while inference starts or applies backpressure. |
| 139 | + if (!chunks.trySend(buffer.copyOf(count)).isSuccess) { |
| 140 | + throw VoiceTranscriptionException( |
| 141 | + "audio-overflow", |
| 142 | + "Speech recognition fell behind capture." |
| 143 | + ) |
| 144 | + } |
| 145 | + bytes += count |
| 146 | + var peak = 0 |
| 147 | + for (i in 0 until count step 2) { |
| 148 | + val sample = ((buffer[i].toInt() and 255) or (buffer[i + 1].toInt() shl 8)).toShort() |
| 149 | + peak = maxOf(peak, abs(sample.toInt())) |
| 150 | + } |
| 151 | + metering = if (peak == 0) -160.0 else 20 * log10(peak / 32768.0) |
| 152 | + } |
| 153 | + } |
| 154 | + } |
| 155 | + |
| 156 | + suspend fun stop() { |
| 157 | + stopping = true |
| 158 | + if (worker == null) { |
| 159 | + finishSource(Result.success(Unit)).getOrThrow() |
| 160 | + output.finish() |
| 161 | + return |
| 162 | + } |
| 163 | + try { |
| 164 | + withTimeout(2000) { microphoneStopped.await().getOrThrow() } |
| 165 | + } catch (error: TimeoutCancellationException) { |
| 166 | + cancel() |
| 167 | + throw VoiceTranscriptionException("incomplete-audio", "The microphone did not stop.", error) |
| 168 | + } |
| 169 | + } |
| 170 | + |
| 171 | + suspend fun awaitFinished() = completion.await().getOrThrow() |
| 172 | + |
| 173 | + fun abort() { |
| 174 | + stopping = true |
| 175 | + worker?.cancel() |
| 176 | + runCatching { stopSource() } |
| 177 | + runCatching { output.finish() } |
| 178 | + if (worker == null) { |
| 179 | + val result = finishSource(Result.failure(CancellationException("Capture was cancelled."))) |
| 180 | + completion.complete(result) |
| 181 | + microphoneStopped.complete(result) |
| 182 | + } |
| 183 | + } |
| 184 | + |
| 185 | + suspend fun cancel() = withContext(NonCancellable) { |
| 186 | + abort() |
| 187 | + worker?.join() |
| 188 | + finishSource(Result.success(Unit)) |
| 189 | + } |
| 190 | + |
| 191 | + @Synchronized private fun stopSource() { |
| 192 | + if (sourceStopped) return |
| 193 | + sourceStopped = true |
| 194 | + source.stop() |
| 195 | + } |
| 196 | + |
| 197 | + @Synchronized private fun finishSource(result: Result<Unit>): Result<Unit> { |
| 198 | + if (sourceReleased) return result |
| 199 | + sourceReleased = true |
| 200 | + var outcome = runCatching { stopSource() }.fold( |
| 201 | + onSuccess = { result }, |
| 202 | + onFailure = { mergeFailure(result, it) } |
| 203 | + ) |
| 204 | + outcome = runCatching { source.release() }.fold( |
| 205 | + onSuccess = { outcome }, |
| 206 | + onFailure = { mergeFailure(outcome, it) } |
| 207 | + ) |
| 208 | + return outcome |
| 209 | + } |
| 210 | +} |
| 211 | + |
| 212 | +private fun mergeFailure(result: Result<Unit>, failure: Throwable): Result<Unit> { |
| 213 | + val original = result.exceptionOrNull() ?: return Result.failure(failure) |
| 214 | + if (original !== failure) original.addSuppressed(failure) |
| 215 | + return result |
| 216 | +} |
0 commit comments