Skip to content

Commit 2530daf

Browse files
asyrovprogsyrovandrewrogerbarreto
authored
.Net: Add .NET VoiceChat console demo: Microphone → VAD → STT → Chat → TTS → Speaker (#12992)
### Motivation and Context Resolves microsoft/semantic-kernel#12916 While the repo already includes an `OpenAIRealtime` demo, this sample shows how **Semantic Kernel** can power a classic, production-style voice pipeline: **VAD → STT → Chat (SK) → TTS → Speaker** This is an end-to-end console app that: - **Uses voice activity detection (VAD)** to skip silence, reducing unnecessary API calls and latency. - **Uses TPL Dataflow** for structured, asynchronous message passing and backpressure, making the flow easier to extend or integrate. - **Streams responses end-to-end**, so the user hears audio as the LLM generates text. - **Runs entirely in a console**, no additional UI frameworks required. This makes the sample representative of real-world voice-agent architecture while remaining minimal and easy to run. Please note that demo is x64 due Web RTC VAD (WebRtcVadSharp) package. --------- Co-authored-by: Andrew Syrov <[email protected]> Co-authored-by: Roger Barreto <[email protected]>
1 parent fb29385 commit 2530daf

20 files changed

Lines changed: 966 additions & 0 deletions

‎dotnet/Directory.Packages.props‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,7 @@
138138
<PackageVersion Include="Microsoft.Extensions.Logging.Console" Version="8.0.1" />
139139
<PackageVersion Include="Microsoft.Extensions.Logging.Debug" Version="8.0.1" />
140140
<PackageVersion Include="Microsoft.Extensions.Options.DataAnnotations" Version="8.0.0" />
141+
<PackageVersion Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="8.0.0" />
141142
<PackageVersion Include="Microsoft.Extensions.TimeProvider.Testing" Version="8.10.0" />
142143
<PackageVersion Include="Microsoft.Extensions.FileProviders.Physical" Version="8.0.0" />
143144
<PackageVersion Include="Microsoft.Extensions.FileProviders.Embedded" Version="8.0.11" />
@@ -230,5 +231,7 @@
230231
<PackageVersion Include="Spectre.Console" Version="0.49.1" />
231232
<PackageVersion Include="Spectre.Console.Cli" Version="0.49.1" />
232233
<PackageVersion Include="Spectre.Console.Json" Version="0.49.1" />
234+
<PackageVersion Include="NAudio" Version="2.2.1" />
235+
<PackageVersion Include="WebRtcVadSharp" Version="1.3.2" />
233236
</ItemGroup>
234237
</Project>

‎dotnet/SK-dotnet.slnx‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,7 @@
9999
<Project Path="samples/Demos/TelemetryWithAppInsights/TelemetryWithAppInsights.csproj" />
100100
<Project Path="samples/Demos/TimePlugin/TimePlugin.csproj" />
101101
<Project Path="samples/Demos/VectorStoreRAG/VectorStoreRAG.csproj" />
102+
<Project Path="samples/Demos/VoiceChat/VoiceChat.csproj" />
102103
</Folder>
103104
<Folder Name="/samples/Demos/A2AClientServer/">
104105
<Project Path="samples/Demos/A2AClientServer/A2AClient/A2AClient.csproj" />
Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
// Copyright (c) Microsoft. All rights reserved.
2+
3+
public class AudioOptions
4+
{
5+
// Audio configuration constants, not part of appsettings, not intended to be changed
6+
public const int SampleRate = 16000;
7+
public const int Channels = 1;
8+
public const int BitsPerSample = 16;
9+
public const int BufferMilliseconds = 20;
10+
}
Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
// Copyright (c) Microsoft. All rights reserved.
2+
3+
using System.ComponentModel.DataAnnotations;
4+
5+
public class ChatOptions
6+
{
7+
public const string SectionName = "Chat";
8+
9+
[Required]
10+
public string SystemMessage { get; set; } = string.Empty;
11+
12+
// Chat response streaming constants
13+
public int StreamingChunkSizeThreshold { get; set; } = 100;
14+
public double Temperature { get; set; } = 0.7;
15+
public int MaxTokens { get; set; } = 500;
16+
public double TopP { get; set; } = 0.9;
17+
}
Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
// Copyright (c) Microsoft. All rights reserved.
2+
3+
using System.ComponentModel.DataAnnotations;
4+
5+
public class OpenAIOptions
6+
{
7+
public const string SectionName = "OpenAI";
8+
9+
[Required]
10+
public string ApiKey { get; set; } = string.Empty;
11+
12+
[Required]
13+
public string ChatModelId { get; set; } = "gpt-4";
14+
15+
[Required]
16+
public string TranscriptionModelId { get; set; } = "gpt-4o-transcribe";
17+
18+
[Required]
19+
public string SpeechModelId { get; set; } = "gpt-4o-mini-tts";
20+
}
Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
// Copyright (c) Microsoft. All rights reserved.
2+
3+
global using AudioChunkEvent = PipelineEvent<byte[]>;
4+
global using AudioEvent = PipelineEvent<AudioData>;
5+
global using ChatEvent = PipelineEvent<string>;
6+
global using SpeechEvent = PipelineEvent<byte[]>;
7+
global using TranscriptionEvent = PipelineEvent<string?>;
8+
9+
public readonly struct PipelineEvent<T>(int turnId, CancellationToken cancellationToken, T payload) : IEquatable<PipelineEvent<T>>
10+
{
11+
public int TurnId { get; } = turnId;
12+
public CancellationToken CancellationToken { get; } = cancellationToken;
13+
public T Payload { get; } = payload;
14+
15+
public static bool IsValid(PipelineEvent<T> evt, int currentTurnId, Func<T, bool>? payloadPredicate = null)
16+
=> evt.Payload != null
17+
&& evt.TurnId == currentTurnId
18+
&& !evt.CancellationToken.IsCancellationRequested
19+
&& (payloadPredicate?.Invoke(evt.Payload) ?? true);
20+
21+
public override bool Equals(object obj)
22+
{
23+
throw new NotImplementedException();
24+
}
25+
26+
public override int GetHashCode()
27+
{
28+
throw new NotImplementedException();
29+
}
30+
31+
public static bool operator ==(PipelineEvent<T> left, PipelineEvent<T> right)
32+
{
33+
return left.Equals(right);
34+
}
35+
36+
public static bool operator !=(PipelineEvent<T> left, PipelineEvent<T> right)
37+
{
38+
return !(left == right);
39+
}
40+
41+
public bool Equals(PipelineEvent<T> other)
42+
{
43+
throw new NotImplementedException();
44+
}
45+
}
46+
47+
public record AudioData(byte[] Data, int SampleRate, int Channels, int BitsPerSample)
48+
{
49+
public TimeSpan Duration => TimeSpan.FromSeconds((double)this.Data.Length / (this.SampleRate * this.Channels * this.BitsPerSample / 8));
50+
}
Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
// Copyright (c) Microsoft. All rights reserved.
2+
3+
public class TurnManager : IDisposable
4+
{
5+
private int _currentTurnId = 0;
6+
private CancellationTokenSource _cts = new();
7+
private readonly object _lock = new();
8+
public int CurrentTurnId { get { lock (this._lock) { return this._currentTurnId; } } }
9+
public CancellationToken CurrentToken { get { lock (this._lock) { return this._cts.Token; } } }
10+
11+
public void Interrupt()
12+
{
13+
lock (this._lock)
14+
{
15+
this._currentTurnId++;
16+
this._cts.Cancel();
17+
this._cts.Dispose();
18+
this._cts = new CancellationTokenSource();
19+
}
20+
}
21+
22+
public void Dispose() => this._cts?.Dispose();
23+
}
Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
// Copyright (c) Microsoft. All rights reserved.
2+
3+
using System.Threading.Tasks.Dataflow;
4+
using Microsoft.Extensions.Logging;
5+
using Microsoft.Extensions.Options;
6+
7+
public class VoiceChatPipeline : IDisposable
8+
{
9+
// Pipeline configuration constants
10+
private const int MaxDegreeOfParallelism = 1; // Number of parallel operations in dataflow blocks
11+
private const int BoundedCapacity = 5; // Maximum capacity for dataflow block buffers
12+
private const bool EnsureOrdered = true; // Ensure order preservation in pipeline
13+
14+
// Dataflow options fields - initialized inline
15+
private readonly ExecutionDataflowBlockOptions _executionOptions = new()
16+
{
17+
MaxDegreeOfParallelism = MaxDegreeOfParallelism,
18+
BoundedCapacity = BoundedCapacity,
19+
EnsureOrdered = EnsureOrdered
20+
};
21+
22+
private readonly DataflowLinkOptions _linkOptions = new() { PropagateCompletion = true };
23+
private readonly ILogger<VoiceChatPipeline> _logger;
24+
private readonly AudioPlaybackService _audioPlaybackService;
25+
private readonly SpeechToTextService _speechToTextService;
26+
private readonly TextToSpeechService _textToSpeechService;
27+
private readonly ChatService _chatService;
28+
private readonly TurnManager _turnManager;
29+
private readonly VadService _vadService;
30+
private readonly AudioSourceService _audioSourceService;
31+
32+
private CancellationTokenSource? _cancellationTokenSource;
33+
34+
public VoiceChatPipeline(
35+
ILogger<VoiceChatPipeline> logger,
36+
AudioPlaybackService audioPlaybackService,
37+
SpeechToTextService speechToTextService,
38+
TextToSpeechService textToSpeechService,
39+
ChatService chatService,
40+
VadService vadService,
41+
AudioSourceService audioSourceService,
42+
TurnManager turnManager,
43+
IOptions<AudioOptions> audioOptions)
44+
{
45+
this._logger = logger;
46+
this._audioPlaybackService = audioPlaybackService;
47+
this._speechToTextService = speechToTextService;
48+
this._textToSpeechService = textToSpeechService;
49+
this._chatService = chatService;
50+
this._vadService = vadService;
51+
this._audioSourceService = audioSourceService;
52+
this._turnManager = turnManager;
53+
}
54+
55+
public async Task RunAsync(CancellationToken cancellationToken = default)
56+
{
57+
this._cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
58+
59+
// Create pipeline blocks - VAD now accepts raw audio chunks directly
60+
var vadBlock = new TransformManyBlock<byte[], AudioEvent>(this._vadService.Transform, this._executionOptions);
61+
var sttBlock = new TransformBlock<AudioEvent, TranscriptionEvent>(this._speechToTextService.TransformAsync, this._executionOptions);
62+
var chatBlock = new TransformManyBlock<TranscriptionEvent, ChatEvent>(this._chatService.TransformAsync, this._executionOptions);
63+
var ttsBlock = new TransformBlock<ChatEvent, SpeechEvent>(this._textToSpeechService.TransformAsync, this._executionOptions);
64+
var playbackBlock = new ActionBlock<SpeechEvent>(this._audioPlaybackService.PipelineActionAsync, this._executionOptions);
65+
66+
// Connect the blocks in the pipeline
67+
this.Link(vadBlock, sttBlock, "VAD", audioData => audioData.Data.Length > 0);
68+
this.Link(sttBlock, chatBlock, "STT", t => !string.IsNullOrEmpty(t));
69+
this.Link(chatBlock, ttsBlock, "Chat", t => !string.IsNullOrEmpty(t));
70+
this.Link(ttsBlock, playbackBlock, "TTS", t => t.Length > 0);
71+
72+
this._logger.LogInformation("Voice Chat started. You can start conversation now, or press Ctrl+C to exit.");
73+
74+
try
75+
{
76+
// Keep feeding audio chunks into the VAD pipeline block till RunAsync is not cancelled
77+
await foreach (var audioChunk in this._audioSourceService.GetAudioChunksAsync(this._cancellationTokenSource.Token))
78+
{
79+
await vadBlock.SendAsync(audioChunk, this._cancellationTokenSource.Token);
80+
}
81+
}
82+
catch (OperationCanceledException)
83+
{
84+
this._logger.LogInformation("Voice Chat pipeline stopping due to cancellation...");
85+
}
86+
finally
87+
{
88+
vadBlock.Complete();
89+
await playbackBlock.Completion;
90+
}
91+
}
92+
93+
public void Dispose()
94+
{
95+
this._vadService?.Dispose();
96+
this._cancellationTokenSource?.Dispose();
97+
}
98+
99+
// Generic filter methods for pipeline events
100+
private bool Filter<T>(PipelineEvent<T> evt, string blockName, Func<T, bool> predicate, IDataflowBlock block)
101+
{
102+
var valid = PipelineEvent<T>.IsValid(evt, this._turnManager.CurrentTurnId, predicate);
103+
if (!valid)
104+
{
105+
this._logger.LogWarning($"{blockName} block: Event filtered out due to cancellation or empty payload.");
106+
}
107+
return valid;
108+
}
109+
110+
private bool FilterDiscarded<T>(PipelineEvent<T> evt, string blockName)
111+
{
112+
this._logger.LogWarning($"{blockName} block: Event filtered out due to cancellation or empty.");
113+
return true;
114+
}
115+
116+
private void Link<T>(
117+
ISourceBlock<PipelineEvent<T>> source,
118+
ITargetBlock<PipelineEvent<T>> target,
119+
string blockName,
120+
Func<T, bool> predicate)
121+
{
122+
source.LinkTo(target, this._linkOptions, evt => this.Filter(evt, blockName, predicate, source));
123+
this.DiscardFiltered(source, blockName);
124+
}
125+
126+
private void DiscardFiltered<T>(ISourceBlock<PipelineEvent<T>> block, string blockName) => block.LinkTo(DataflowBlock.NullTarget<PipelineEvent<T>>(), this._linkOptions, evt => this.FilterDiscarded(evt, blockName));
127+
}
Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
// Copyright (c) Microsoft. All rights reserved.
2+
3+
using Microsoft.Extensions.Configuration;
4+
using Microsoft.Extensions.DependencyInjection;
5+
using Microsoft.Extensions.Hosting;
6+
using Microsoft.SemanticKernel;
7+
8+
internal static class Program
9+
{
10+
internal static async Task Main(string[] args)
11+
{
12+
var builder = Host.CreateApplicationBuilder(args);
13+
builder.Configuration.AddUserSecrets<OpenAIOptions>();
14+
// Adding configuration from appsettings.json and environment variables
15+
builder.Services.ConfigureOptions<OpenAIOptions>(OpenAIOptions.SectionName);
16+
builder.Services.ConfigureOptions<ChatOptions>(ChatOptions.SectionName);
17+
18+
// Configure Semantic Kernel in DI container
19+
builder.Services
20+
.AddKernel()
21+
.AddOpenAIChatCompletion(
22+
modelId: builder.Configuration[$"{OpenAIOptions.SectionName}:ChatModelId"]!,
23+
apiKey: builder.Configuration[$"{OpenAIOptions.SectionName}:ApiKey"]!
24+
);
25+
26+
// Register audio chat pipeline services
27+
builder.Services.AddSingleton<AudioPlaybackService>();
28+
builder.Services.AddSingleton<SpeechToTextService>();
29+
builder.Services.AddSingleton<TextToSpeechService>();
30+
builder.Services.AddSingleton<ChatService>();
31+
builder.Services.AddSingleton<TurnManager>();
32+
builder.Services.AddSingleton<VadService>();
33+
builder.Services.AddSingleton<AudioSourceService>();
34+
35+
// Register audio chat pipeline
36+
builder.Services.AddTransient<VoiceChatPipeline>();
37+
38+
using var host = builder.Build();
39+
40+
// Setting up graceful shutdown on Ctrl+C
41+
using var cts = new CancellationTokenSource();
42+
Console.CancelKeyPress += (_, e) =>
43+
{
44+
e.Cancel = true;
45+
cts.Cancel();
46+
};
47+
48+
// Run the voice chat pipeline
49+
using var pipeline = host.Services.GetRequiredService<VoiceChatPipeline>();
50+
await pipeline.RunAsync(cts.Token);
51+
}
52+
53+
private static void ConfigureOptions<TOptions>(this IServiceCollection services, string sectionName) where TOptions : class =>
54+
services
55+
.AddOptions<TOptions>()
56+
.BindConfiguration(sectionName)
57+
.ValidateDataAnnotations()
58+
.ValidateOnStart();
59+
}
Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
# Voice Chat
2+
3+
This sample demonstrates a simple voice chat application built with Semantic Kernel and OpenAI’s API for speech-to-text (STT), chat completion, and text-to-speech (TTS).
4+
5+
It captures audio from the microphone, processes it through a pipeline, and plays back the AI-generated responses:
6+
7+
Microphone → VAD → STT → Chat (Semantic Kernel) → TTS → Speaker
8+
9+
## Purpose
10+
11+
This is not a complete application, but a **starting point** that shows how an audio pipeline can be built using Semantic Kernel and the .NET DataFlow library.
12+
It’s intended to help you understand how to structure audio processing with SK, rather than provide a production-ready chat app.
13+
14+
## Voice Activity Detection
15+
16+
This demo uses **WebRTC VAD** to detect when the user starts and stops speaking.
17+
Other model-based approaches can also be used, such as **[Silero VAD](https://github.com/snakers4/silero-vad/tree/master/examples/csharp)**, which may provide higher accuracy.
18+
19+
## Known Limitations
20+
21+
- **Latency**
22+
This demo processes audio in discrete steps (non-streaming). Response times are therefore large, sometimes over 20 seconds.
23+
To reduce latency, you should use **streaming STT and TTS services** (see below).
24+
25+
- **OpenAI Free Tier Rate Limits**
26+
Very high latencies can also be caused by OpenAI’s rate limits, especially on free-tier accounts. See the OpenAI [rate limits documentation](https://platform.openai.com/docs/guides/rate-limits) for more details.
27+
28+
- **Latency Resources**
29+
For more on latency in voice AI pipelines, see this resource: [Latency in LLM Voice Pipelines](https://voiceaiandvoiceagents.com/#latency-llm).
30+
31+
## Suggested Streaming Services
32+
33+
To reduce latency in real-world scenarios, you can integrate with streaming speech services such as:
34+
35+
- **Speech-to-Text (STT)**
36+
- [OpenAI Realtime API (Whisper streaming)](https://platform.openai.com/docs/guides/realtime)
37+
- [Azure Cognitive Services Speech-to-Text](https://learn.microsoft.com/azure/cognitive-services/speech-service/speech-to-text)
38+
- [Deepgram Streaming STT](https://developers.deepgram.com/docs/streaming)
39+
- [AssemblyAI Streaming STT](https://www.assemblyai.com/docs/real-time-speech-recognition)
40+
41+
- **Text-to-Speech (TTS)**
42+
- [OpenAI Realtime API (TTS streaming)](https://platform.openai.com/docs/guides/realtime)
43+
- [Azure Cognitive Services Text-to-Speech](https://learn.microsoft.com/azure/cognitive-services/speech-service/text-to-speech)
44+
- [Amazon Polly Neural TTS](https://docs.aws.amazon.com/polly/latest/dg/what-is.html)
45+
46+
## How to Run
47+
48+
1. Store your API key securely with [.NET user-secrets](https://learn.microsoft.com/aspnet/core/security/app-secrets):
49+
50+
dotnet user-secrets set "OpenAI:ApiKey" "your-openai-api-key"
51+
52+
2. Build and run the sample:
53+
54+
dotnet run --project samples/Demos/VoiceChat
55+
56+
3. Speak into your microphone and listen for the AI response through your speakers.

0 commit comments

Comments
 (0)