Skip to content

Commit 8a3e86f

Browse files
mayoalexanderclaude
andcommitted
feat(chat): iris chat --voice — free local real-time voice for any agent
Wire the two primitives IRIS already shipped but never connected: on-device whisper.cpp STT (lib/transcription.transcribeLocal, already used by `ideas capture`) + the V6 ReactLoop chat brain (streamAgentChat). Adds the missing local TTS + push-to-talk turn-taking so you can hold a spoken conversation with any agent (e.g. TOBI anomalyco#642) fully on-device — $0/turn, offline-capable, HIPAA-safe. Cloud voices (ElevenLabs/VAPI) stay in `iris voice` for phone. - lib/voice.ts: captureMic (ffmpeg push-to-talk → 16kHz WAV), speak (macOS `say` default, Piper optional, never throws), listMics (device discovery). - chat: --voice loop with multi-turn conversation_history (no server session, no cross-turn poisoning); flags --mic / --tts / --tts-voice / --list-mics. Plan + tracking: bloq #503 item #158044, bug #158045. Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
1 parent 347dcf5 commit 8a3e86f

2 files changed

Lines changed: 336 additions & 0 deletions

File tree

‎packages/opencode/src/cli/cmd/platform-chat.ts‎

Lines changed: 184 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ import { cmd } from "./cmd"
22
import * as prompts from "./clack"
33
import { UI } from "../ui"
44
import { irisFetch, requireAuth, handleApiError, printDivider, dim, bold, FL_API, IRIS_API, resolveUserId, streamAgentChat } from "./iris-api"
5+
import { captureMic, speak, listMics } from "../lib/voice"
6+
import { transcribeLocal, which } from "../lib/transcription"
57

68
// ============================================================================
79
// Polling helper
@@ -231,6 +233,137 @@ export async function executeChat(args: {
231233
}
232234
}
233235

236+
// ============================================================================
237+
// Voice chat — local, free, real-time conversation loop.
238+
//
239+
// mic (ffmpeg) → transcribeLocal (whisper.cpp) → streamAgentChat → speak (say).
240+
// Push-to-talk turn-taking; multi-turn via conversation_history (no server
241+
// session, so no cross-turn poisoning — same guarantee as text chat). All STT
242+
// + TTS runs on-device; only the agent call leaves the machine. (#158044/#158045)
243+
// ============================================================================
244+
245+
export async function runVoiceChat(args: {
246+
agent?: number
247+
bloq?: number
248+
timeout: number
249+
"no-rag": boolean
250+
model?: string
251+
"max-iterations"?: number
252+
mic?: string
253+
tts?: string
254+
"tts-voice"?: string
255+
}): Promise<void> {
256+
UI.empty()
257+
prompts.intro("◈ IRIS Voice Chat")
258+
259+
const token = await requireAuth()
260+
if (!token) { prompts.outro("Done"); return }
261+
262+
const agentId = args.agent
263+
if (!agentId) {
264+
prompts.log.warn("Voice chat needs an explicit agent. Use --agent <id>.")
265+
prompts.log.info(`Try: ${dim("iris agents list")} to find one (e.g. --agent 642 for TOBI)`)
266+
prompts.outro("Done")
267+
return
268+
}
269+
270+
if (!which("ffmpeg") || (!which("whisper-cli") && !which("whisper-cpp"))) {
271+
prompts.log.error("Local voice needs ffmpeg + whisper-cpp.")
272+
prompts.log.info(`Install: ${dim("brew install ffmpeg whisper-cpp")}`)
273+
prompts.outro("Done")
274+
return
275+
}
276+
277+
const userId = await resolveUserId()
278+
const mics = listMics()
279+
const micLabel = args.mic
280+
? mics.find((m) => m.index === args.mic)?.name ?? `device :${args.mic}`
281+
: "system default"
282+
const ttsLabel = args.tts ?? (process.platform === "darwin" ? "say" : "piper")
283+
284+
prompts.log.info(`${bold(`Agent #${agentId}`)} ${dim(`· mic: ${micLabel} · tts: ${ttsLabel}`)}`)
285+
prompts.log.info(dim("ENTER = speak · ENTER again = stop · type q + ENTER to quit"))
286+
287+
const history: Array<{ role: string; content: string }> = []
288+
289+
while (true) {
290+
process.stderr.write(`\n ${dim("▶︎ press ENTER to speak (q to quit)…")} `)
291+
const key = await new Promise<string>((resolve) => {
292+
process.stdin.resume()
293+
const onData = (d: Buffer) => {
294+
process.stdin.removeListener("data", onData)
295+
process.stdin.pause()
296+
resolve(d.toString().trim())
297+
}
298+
process.stdin.once("data", onData)
299+
})
300+
if (key.toLowerCase() === "q") break
301+
302+
// Capture + transcribe (both on-device).
303+
let text = ""
304+
try {
305+
const wav = await captureMic({ mic: args.mic })
306+
process.stderr.write("\r" + " ".repeat(48) + "\r")
307+
text = await transcribeLocal(wav)
308+
} catch (err) {
309+
prompts.log.error(err instanceof Error ? err.message : String(err))
310+
continue
311+
}
312+
if (!text) { prompts.log.info(dim("(heard nothing — try again)")); continue }
313+
console.log(` ${bold("You:")} ${text}`)
314+
315+
// Ask the agent (same faithful V6 ReactLoop path as text chat).
316+
history.push({ role: "user", content: text })
317+
const startTime = Date.now()
318+
const spinner = prompts.spinner()
319+
spinner.start("Thinking…")
320+
let lastActivity = "Thinking…"
321+
const heartbeat = setInterval(() => {
322+
spinner.message(`${lastActivity} (${Math.floor((Date.now() - startTime) / 1000)}s)`)
323+
}, 1000)
324+
325+
try {
326+
const result = await streamAgentChat({
327+
agentId,
328+
message: text,
329+
userId,
330+
bloqId: args.bloq,
331+
overrideModel: args.model,
332+
maxIterations: args["max-iterations"],
333+
timeoutSecs: args.timeout,
334+
enableRag: !args["no-rag"],
335+
conversationHistory: history.slice(0, -1),
336+
onEvent: (evt) => {
337+
if (evt.type === "tool_call" && evt.tool) lastActivity = `Using ${evt.tool}…`
338+
else if (evt.type === "tool_result" && evt.tool) lastActivity = `${evt.tool} ✓`
339+
else if (evt.type === "thinking") lastActivity = "Thinking…"
340+
},
341+
})
342+
clearInterval(heartbeat)
343+
344+
if (!result.ok) {
345+
spinner.stop(result.timedOut ? "Timed out" : "Failed", 1)
346+
prompts.log.error(result.error ?? "Chat failed — no answer delivered.")
347+
history.pop() // drop the unanswered turn so history stays consistent
348+
continue
349+
}
350+
351+
spinner.stop(dim(`${result.iterations ?? 0} iter · ${((Date.now() - startTime) / 1000).toFixed(1)}s`))
352+
const reply = result.content || "(no response)"
353+
history.push({ role: "assistant", content: reply })
354+
console.log(` ${bold("Agent:")} ${reply.split("\n").join("\n ")}`)
355+
await speak(reply, { tts: args.tts, voice: args["tts-voice"] })
356+
} catch (err) {
357+
clearInterval(heartbeat)
358+
spinner.stop("Error", 1)
359+
prompts.log.error(err instanceof Error ? err.message : String(err))
360+
history.pop()
361+
}
362+
}
363+
364+
prompts.outro("Voice chat ended 👋")
365+
}
366+
234367
function outputResult(run: WorkflowRun, workflowId: string, agentId: number | undefined, isJson: boolean, toolsUsed: string[] = []): void {
235368
const response = run.summary ?? run.response ?? run.output ?? "(no response)"
236369

@@ -427,9 +560,60 @@ export const PlatformChatCommand = cmd({
427560
describe: "cap ReactLoop iterations",
428561
type: "number",
429562
})
563+
.option("voice", {
564+
describe: "voice mode — talk to the agent via mic + local speech (free, on-device)",
565+
type: "boolean",
566+
default: false,
567+
})
568+
.option("mic", {
569+
describe: "input device (macOS index from --list-mics, or ALSA name); default = system mic",
570+
type: "string",
571+
})
572+
.option("tts", {
573+
describe: "speech backend for replies: say | piper | none",
574+
type: "string",
575+
choices: ["say", "piper", "none"],
576+
})
577+
.option("tts-voice", {
578+
describe: "TTS voice name (e.g. macOS `say -v` voice)",
579+
type: "string",
580+
})
581+
.option("list-mics", {
582+
describe: "list available input devices and exit",
583+
type: "boolean",
584+
default: false,
585+
})
430586
.command(ChatApproveCommand),
431587

432588
async handler(args) {
589+
if (args["list-mics"]) {
590+
UI.empty()
591+
prompts.intro("◈ IRIS Voice — Input Devices")
592+
const mics = listMics()
593+
if (mics.length === 0) {
594+
prompts.log.info("No devices enumerated (device listing is macOS-only; on Linux pass --mic <alsa-name>).")
595+
} else {
596+
for (const m of mics) console.log(` ${bold(`:${m.index}`)} ${m.name}`)
597+
}
598+
prompts.outro(`Use: ${dim("iris chat --voice --agent <id> --mic <index>")}`)
599+
return
600+
}
601+
602+
if (args.voice) {
603+
await runVoiceChat({
604+
agent: args.agent,
605+
bloq: args.bloq,
606+
timeout: args.timeout,
607+
"no-rag": args["no-rag"],
608+
model: args.model,
609+
"max-iterations": args["max-iterations"],
610+
mic: args.mic,
611+
tts: args.tts,
612+
"tts-voice": args["tts-voice"],
613+
})
614+
return
615+
}
616+
433617
if (!args.message && !args.continue) {
434618
UI.empty()
435619
prompts.intro("◈ IRIS Chat")
Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
import { spawn, spawnSync } from "child_process"
2+
import { existsSync } from "fs"
3+
import { tmpdir } from "os"
4+
import { join } from "path"
5+
import { which } from "./transcription"
6+
7+
// ============================================================================
8+
// Voice lib — the local, free, on-device half of voice chat.
9+
//
10+
// captureMic() = push-to-talk mic capture via ffmpeg → 16kHz mono WAV.
11+
// Pairs with transcribeLocal() (whisper.cpp) for STT.
12+
// speak() = local text-to-speech. macOS `say` (zero-dep default) or
13+
// Piper (cross-platform neural) — no cloud, no per-minute cost.
14+
// listMics() = enumerate input devices so `--mic <id>` is discoverable.
15+
//
16+
// Everything here runs on-device: HIPAA-safe, offline-capable, $0 per turn.
17+
// Cloud voices (ElevenLabs/VAPI) stay in `iris voice` for phone/agent config.
18+
// ============================================================================
19+
20+
export interface Mic {
21+
index: string
22+
name: string
23+
}
24+
25+
/** Enumerate audio input devices (macOS avfoundation). Empty on other platforms. */
26+
export function listMics(): Mic[] {
27+
if (process.platform !== "darwin") return []
28+
const r = spawnSync("ffmpeg", ["-f", "avfoundation", "-list_devices", "true", "-i", ""], { encoding: "utf8" })
29+
const out = r.stderr || ""
30+
const mics: Mic[] = []
31+
let inAudio = false
32+
for (const line of out.split("\n")) {
33+
if (/AVFoundation audio devices/i.test(line)) { inAudio = true; continue }
34+
if (/AVFoundation video devices/i.test(line)) { inAudio = false; continue }
35+
if (!inAudio) continue
36+
const m = line.match(/\[(\d+)\]\s+(.+?)\s*$/)
37+
if (m) mics.push({ index: m[1], name: m[2].trim() })
38+
}
39+
return mics
40+
}
41+
42+
/** Platform-specific ffmpeg input args. `mic` is a device index (macOS) or ALSA name (linux). */
43+
function micInputArgs(mic?: string): string[] {
44+
switch (process.platform) {
45+
case "darwin":
46+
// `:default` follows the system input device; `:<index>` pins a specific mic.
47+
return ["-f", "avfoundation", "-i", `:${mic ?? "default"}`]
48+
case "linux":
49+
return ["-f", "alsa", "-i", mic ?? "default"]
50+
default:
51+
throw new Error(`Voice capture not supported on ${process.platform} yet — use --text or macOS/Linux.`)
52+
}
53+
}
54+
55+
/** Block until the user presses ENTER; resolves with the trimmed line typed (for "q" to quit). */
56+
function readLine(): Promise<string> {
57+
return new Promise((resolve) => {
58+
process.stdin.resume()
59+
const onData = (d: Buffer) => {
60+
process.stdin.removeListener("data", onData)
61+
process.stdin.pause()
62+
resolve(d.toString().trim())
63+
}
64+
process.stdin.once("data", onData)
65+
})
66+
}
67+
68+
/**
69+
* Push-to-talk mic capture. Records from ENTER (already pressed by caller) until
70+
* the user presses ENTER again, then finalizes a 16kHz mono WAV and returns its
71+
* path. `-nostdin` keeps ffmpeg off the terminal so our own ENTER read wins.
72+
*/
73+
export async function captureMic(opts: { mic?: string } = {}): Promise<string> {
74+
const ffmpeg = which("ffmpeg")
75+
if (!ffmpeg) throw new Error("ffmpeg not found. Install: brew install ffmpeg")
76+
77+
const wav = join(tmpdir(), `iris-voice-${Date.now()}.wav`)
78+
const args = [
79+
"-hide_banner", "-loglevel", "error", "-nostdin", "-y",
80+
...micInputArgs(opts.mic),
81+
"-ar", "16000", "-ac", "1", "-c:a", "pcm_s16le", wav,
82+
]
83+
const proc = spawn(ffmpeg, args, { stdio: ["ignore", "ignore", "ignore"] })
84+
85+
process.stderr.write(" 🔴 recording… press ENTER to stop ")
86+
await readLine()
87+
88+
// SIGINT lets ffmpeg write the WAV trailer cleanly (a hard kill truncates it).
89+
proc.kill("SIGINT")
90+
await new Promise<void>((resolve) => {
91+
proc.on("close", () => resolve())
92+
proc.on("error", () => resolve())
93+
})
94+
return wav
95+
}
96+
97+
/** Strip markdown so TTS doesn't read asterisks/backticks/link syntax aloud. */
98+
function stripForSpeech(text: string): string {
99+
return text
100+
.replace(/```[\s\S]*?```/g, " code block ")
101+
.replace(/`([^`]+)`/g, "$1")
102+
.replace(/\*\*([^*]+)\*\*/g, "$1")
103+
.replace(/\*([^*]+)\*/g, "$1")
104+
.replace(/^#+\s*/gm, "")
105+
.replace(/\[([^\]]+)\]\([^)]+\)/g, "$1")
106+
.replace(/[_>]/g, "")
107+
.trim()
108+
}
109+
110+
/**
111+
* Speak text locally. tts: "say" (macOS), "piper" (neural, needs IRIS_PIPER_MODEL),
112+
* or "none". Falls back to `say` on macOS if the requested backend is unavailable.
113+
* Never throws — a failed TTS must not kill the conversation loop.
114+
*/
115+
export async function speak(text: string, opts: { tts?: string; voice?: string } = {}): Promise<void> {
116+
const clean = stripForSpeech(text)
117+
if (!clean) return
118+
let tts = opts.tts || (process.platform === "darwin" ? "say" : "piper")
119+
if (tts === "none") return
120+
121+
const run = (bin: string, args: string[], input?: Buffer): Promise<void> =>
122+
new Promise((resolve) => {
123+
const p = spawn(bin, args, { stdio: [input ? "pipe" : "ignore", "ignore", "ignore"] })
124+
p.on("close", () => resolve())
125+
p.on("error", () => resolve())
126+
if (input) { p.stdin?.write(input); p.stdin?.end() }
127+
})
128+
129+
if (tts === "piper") {
130+
const piper = which("piper")
131+
const model = process.env.IRIS_PIPER_MODEL
132+
const player = which("afplay") || which("ffplay")
133+
if (piper && model && existsSync(model) && player) {
134+
const wav = join(tmpdir(), `iris-tts-${Date.now()}.wav`)
135+
await run(piper, ["-m", model, "-f", wav], Buffer.from(clean))
136+
if (existsSync(wav)) {
137+
await run(player, player.endsWith("ffplay") ? ["-nodisp", "-autoexit", "-loglevel", "quiet", wav] : [wav])
138+
spawnSync("rm", ["-f", wav])
139+
return
140+
}
141+
}
142+
// Piper not ready → fall back to say on macOS, else silent.
143+
tts = process.platform === "darwin" ? "say" : "none"
144+
if (tts === "none") return
145+
}
146+
147+
if (tts === "say") {
148+
const say = which("say")
149+
if (!say) return
150+
await run(say, opts.voice ? ["-v", opts.voice, clean] : [clean])
151+
}
152+
}

0 commit comments

Comments
 (0)