Skip to content

Commit ef85b61

Browse files
rogerbarretodedaloshawncal
authored
IChatCompletion Streaming Support (#829)
### Motivation and Context Get streaming results from chat completion models using the IAsyncEnumerable pattern. Closes #378 Resolves #378 Added samples how to use those as well as how to implement your own custom chat completion model using streaming capability. Co-authored-by: dedalo <[email protected]> Co-authored-by: Shawn Callegari <[email protected]>
1 parent e38d97f commit ef85b61

10 files changed

Lines changed: 507 additions & 86 deletions

File tree

‎dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/ClientBase.cs‎

Lines changed: 101 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ protected async Task<string> InternalCompleteTextAsync(
4444
Verify.NotNull(requestSettings);
4545

4646
ValidateMaxTokens(requestSettings.MaxTokens);
47-
var options = this.CreateCompletionsOptions(text, requestSettings);
47+
var options = CreateCompletionsOptions(text, requestSettings);
4848

4949
Response<Completions>? response = await RunRequestAsync<Response<Completions>?>(
5050
() => this.Client.GetCompletionsAsync(this.ModelId, options, cancellationToken)).ConfigureAwait(false);
@@ -72,7 +72,7 @@ protected async IAsyncEnumerable<string> InternalCompletionStreamAsync(
7272
Verify.NotNull(requestSettings);
7373

7474
ValidateMaxTokens(requestSettings.MaxTokens);
75-
var options = this.CreateCompletionsOptions(text, requestSettings);
75+
var options = CreateCompletionsOptions(text, requestSettings);
7676

7777
Response<StreamingCompletions>? response = await RunRequestAsync<Response<StreamingCompletions>>(
7878
() => this.Client.GetCompletionsStreamingAsync(this.ModelId, options, cancellationToken)).ConfigureAwait(false);
@@ -135,31 +135,35 @@ protected async Task<string> InternalGenerateChatMessageAsync(
135135
Verify.NotNull(chat);
136136
Verify.NotNull(requestSettings);
137137

138-
if (requestSettings.MaxTokens < 1)
139-
{
140-
throw new AIException(
141-
AIException.ErrorCodes.InvalidRequest,
142-
$"MaxTokens {requestSettings.MaxTokens} is not valid, the value must be greater than zero");
143-
}
138+
ValidateMaxTokens(requestSettings.MaxTokens);
139+
var options = CreateChatCompletionsOptions(requestSettings, chat);
144140

145-
var options = new ChatCompletionsOptions
146-
{
147-
MaxTokens = requestSettings.MaxTokens,
148-
Temperature = (float?)requestSettings.Temperature,
149-
NucleusSamplingFactor = (float?)requestSettings.TopP,
150-
FrequencyPenalty = (float?)requestSettings.FrequencyPenalty,
151-
PresencePenalty = (float?)requestSettings.PresencePenalty,
152-
ChoicesPerPrompt = 1,
153-
};
141+
Response<ChatCompletions>? response = await RunRequestAsync<Response<ChatCompletions>?>(
142+
() => this.Client.GetChatCompletionsAsync(this.ModelId, options, cancellationToken)).ConfigureAwait(false);
154143

155-
if (requestSettings.StopSequences is { Count: > 0 })
144+
if (response == null || response.Value.Choices.Count < 1)
156145
{
157-
foreach (var s in requestSettings.StopSequences)
158-
{
159-
options.StopSequences.Add(s);
160-
}
146+
throw new AIException(AIException.ErrorCodes.InvalidResponseContent, "Chat completions not found");
161147
}
162148

149+
return response.Value.Choices[0].Message.Content;
150+
}
151+
152+
/// <summary>
153+
/// Generate a new chat message stream
154+
/// </summary>
155+
/// <param name="chat">Chat history</param>
156+
/// <param name="requestSettings">AI request settings</param>
157+
/// <param name="cancellationToken">Async cancellation token</param>
158+
/// <returns>Streaming of generated chat message in string format</returns>
159+
protected async IAsyncEnumerable<string> InternalGenerateChatMessageStreamAsync(
160+
ChatHistory chat,
161+
ChatRequestSettings requestSettings,
162+
[EnumeratorCancellation] CancellationToken cancellationToken = default)
163+
{
164+
Verify.NotNull(chat);
165+
Verify.NotNull(requestSettings);
166+
163167
foreach (ChatHistory.Message message in chat.Messages)
164168
{
165169
var role = message.AuthorRole switch
@@ -169,27 +173,38 @@ protected async Task<string> InternalGenerateChatMessageAsync(
169173
ChatHistory.AuthorRoles.System => ChatRole.System,
170174
_ => throw new ArgumentException($"Invalid chat message author: {message.AuthorRole:G}")
171175
};
172-
173-
options.Messages.Add(new ChatMessage(role, message.Content));
174176
}
175177

176-
Response<ChatCompletions>? response = await RunRequestAsync<Response<ChatCompletions>?>(
177-
() => this.Client.GetChatCompletionsAsync(this.ModelId, options, cancellationToken)).ConfigureAwait(false);
178+
ValidateMaxTokens(requestSettings.MaxTokens);
179+
var options = CreateChatCompletionsOptions(requestSettings, chat);
178180

179-
if (response == null || response.Value.Choices.Count < 1)
181+
Response<StreamingChatCompletions>? response = await RunRequestAsync<Response<StreamingChatCompletions>>(
182+
() => this.Client.GetChatCompletionsStreamingAsync(this.ModelId, options, cancellationToken)).ConfigureAwait(false);
183+
184+
using StreamingChatCompletions streamingChatCompletions = response.Value;
185+
186+
if (response is null)
180187
{
181188
throw new AIException(AIException.ErrorCodes.InvalidResponseContent, "Chat completions not found");
182189
}
183190

184-
return response.Value.Choices[0].Message.Content;
191+
await foreach (StreamingChatChoice choice in streamingChatCompletions.GetChoicesStreaming(cancellationToken))
192+
{
193+
await foreach (ChatMessage message in choice.GetMessageStreaming(cancellationToken))
194+
{
195+
yield return message.Content;
196+
}
197+
198+
yield return Environment.NewLine;
199+
}
185200
}
186201

187202
/// <summary>
188203
/// Create a new empty chat instance
189204
/// </summary>
190205
/// <param name="instructions">Optional chat instructions for the AI service</param>
191206
/// <returns>Chat object</returns>
192-
protected ChatHistory InternalCreateNewChat(string instructions = "")
207+
protected static ChatHistory InternalCreateNewChat(string instructions = "")
193208
{
194209
return new OpenAIChatHistory(instructions);
195210
}
@@ -206,9 +221,26 @@ protected async Task<string> InternalCompleteTextUsingChatAsync(
206221
CompleteRequestSettings requestSettings,
207222
CancellationToken cancellationToken = default)
208223
{
209-
var chat = this.InternalCreateNewChat();
224+
ChatHistory chat = PrepareChatHistory(text, requestSettings, out ChatRequestSettings settings);
225+
226+
return await this.InternalGenerateChatMessageAsync(chat, settings, cancellationToken).ConfigureAwait(false);
227+
}
228+
229+
protected IAsyncEnumerable<string> InternalCompleteTextUsingChatStreamAsync(
230+
string text,
231+
CompleteRequestSettings requestSettings,
232+
CancellationToken cancellationToken = default)
233+
{
234+
ChatHistory chat = PrepareChatHistory(text, requestSettings, out ChatRequestSettings settings);
235+
236+
return this.InternalGenerateChatMessageStreamAsync(chat, settings, cancellationToken);
237+
}
238+
239+
private static ChatHistory PrepareChatHistory(string text, CompleteRequestSettings requestSettings, out ChatRequestSettings settings)
240+
{
241+
var chat = InternalCreateNewChat();
210242
chat.AddMessage(ChatHistory.AuthorRoles.User, text);
211-
var settings = new ChatRequestSettings
243+
settings = new ChatRequestSettings
212244
{
213245
MaxTokens = requestSettings.MaxTokens,
214246
Temperature = requestSettings.Temperature,
@@ -217,11 +249,10 @@ protected async Task<string> InternalCompleteTextUsingChatAsync(
217249
FrequencyPenalty = requestSettings.FrequencyPenalty,
218250
StopSequences = requestSettings.StopSequences,
219251
};
220-
221-
return await this.InternalGenerateChatMessageAsync(chat, settings, cancellationToken).ConfigureAwait(false);
252+
return chat;
222253
}
223254

224-
private CompletionsOptions CreateCompletionsOptions(string text, CompleteRequestSettings requestSettings)
255+
private static CompletionsOptions CreateCompletionsOptions(string text, CompleteRequestSettings requestSettings)
225256
{
226257
var options = new CompletionsOptions
227258
{
@@ -249,6 +280,42 @@ private CompletionsOptions CreateCompletionsOptions(string text, CompleteRequest
249280
return options;
250281
}
251282

283+
private static ChatCompletionsOptions CreateChatCompletionsOptions(ChatRequestSettings requestSettings, ChatHistory chat)
284+
{
285+
var options = new ChatCompletionsOptions
286+
{
287+
MaxTokens = requestSettings.MaxTokens,
288+
Temperature = (float?)requestSettings.Temperature,
289+
NucleusSamplingFactor = (float?)requestSettings.TopP,
290+
FrequencyPenalty = (float?)requestSettings.FrequencyPenalty,
291+
PresencePenalty = (float?)requestSettings.PresencePenalty,
292+
ChoicesPerPrompt = 1,
293+
};
294+
295+
if (requestSettings.StopSequences is { Count: > 0 })
296+
{
297+
foreach (var s in requestSettings.StopSequences)
298+
{
299+
options.StopSequences.Add(s);
300+
}
301+
}
302+
303+
foreach (ChatHistory.Message message in chat.Messages)
304+
{
305+
var role = message.AuthorRole switch
306+
{
307+
ChatHistory.AuthorRoles.User => ChatRole.User,
308+
ChatHistory.AuthorRoles.Assistant => ChatRole.Assistant,
309+
ChatHistory.AuthorRoles.System => ChatRole.System,
310+
_ => throw new ArgumentException($"Invalid chat message author: {message.AuthorRole:G}")
311+
};
312+
313+
options.Messages.Add(new ChatMessage(role, message.Content));
314+
}
315+
316+
return options;
317+
}
318+
252319
private static void ValidateMaxTokens(int maxTokens)
253320
{
254321
if (maxTokens < 1)

‎dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/AzureChatCompletion.cs‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
// Copyright (c) Microsoft. All rights reserved.
22

33
using System.Collections.Generic;
4-
using System.Linq;
54
using System.Net.Http;
65
using System.Threading;
76
using System.Threading.Tasks;
@@ -62,10 +61,19 @@ public Task<string> GenerateMessageAsync(
6261
return this.InternalGenerateChatMessageAsync(chat, requestSettings ?? new(), cancellationToken);
6362
}
6463

64+
/// <inheritdoc/>
65+
public IAsyncEnumerable<string> GenerateMessageStreamAsync(
66+
ChatHistory chat,
67+
ChatRequestSettings? requestSettings = null,
68+
CancellationToken cancellationToken = default)
69+
{
70+
return this.InternalGenerateChatMessageStreamAsync(chat, requestSettings ?? new(), cancellationToken);
71+
}
72+
6573
/// <inheritdoc/>
6674
public ChatHistory CreateNewChat(string instructions = "")
6775
{
68-
return this.InternalCreateNewChat(instructions);
76+
return InternalCreateNewChat(instructions);
6977
}
7078

7179
/// <inheritdoc/>
@@ -82,6 +90,6 @@ public IAsyncEnumerable<string> CompleteStreamAsync(string text,
8290
CompleteRequestSettings requestSettings,
8391
CancellationToken cancellationToken = default)
8492
{
85-
return this.InternalCompleteTextUsingChatAsync(text, requestSettings, cancellationToken).ToAsyncEnumerable();
93+
return this.InternalCompleteTextUsingChatStreamAsync(text, requestSettings, cancellationToken);
8694
}
8795
}

‎dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/OpenAIChatCompletion.cs‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
// Copyright (c) Microsoft. All rights reserved.
22

33
using System.Collections.Generic;
4-
using System.Linq;
54
using System.Net.Http;
65
using System.Threading;
76
using System.Threading.Tasks;
@@ -42,15 +41,22 @@ public Task<string> GenerateMessageAsync(
4241
ChatRequestSettings? requestSettings = null,
4342
CancellationToken cancellationToken = default)
4443
{
45-
requestSettings ??= new ChatRequestSettings();
44+
return this.InternalGenerateChatMessageAsync(chat, requestSettings ?? new(), cancellationToken);
45+
}
4646

47-
return this.InternalGenerateChatMessageAsync(chat, requestSettings, cancellationToken);
47+
/// <inheritdoc/>
48+
public IAsyncEnumerable<string> GenerateMessageStreamAsync(
49+
ChatHistory chat,
50+
ChatRequestSettings? requestSettings = null,
51+
CancellationToken cancellationToken = default)
52+
{
53+
return this.InternalGenerateChatMessageStreamAsync(chat, requestSettings ?? new(), cancellationToken);
4854
}
4955

5056
/// <inheritdoc/>
5157
public ChatHistory CreateNewChat(string instructions = "")
5258
{
53-
return this.InternalCreateNewChat(instructions);
59+
return InternalCreateNewChat(instructions);
5460
}
5561

5662
/// <inheritdoc/>
@@ -68,6 +74,6 @@ public IAsyncEnumerable<string> CompleteStreamAsync(
6874
CompleteRequestSettings requestSettings,
6975
CancellationToken cancellationToken = default)
7076
{
71-
return this.InternalCompleteTextUsingChatAsync(text, requestSettings, cancellationToken).ToAsyncEnumerable();
77+
return this.InternalCompleteTextUsingChatStreamAsync(text, requestSettings, cancellationToken);
7278
}
7379
}

‎dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/IChatCompletion.cs‎

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,20 @@
11
// Copyright (c) Microsoft. All rights reserved.
22

3+
using System.Collections.Generic;
34
using System.Threading;
45
using System.Threading.Tasks;
56

67
namespace Microsoft.SemanticKernel.AI.ChatCompletion;
78

89
public interface IChatCompletion
910
{
11+
/// <summary>
12+
/// Create a new empty chat instance
13+
/// </summary>
14+
/// <param name="instructions">Optional chat instructions for the AI service</param>
15+
/// <returns>Chat object</returns>
16+
public ChatHistory CreateNewChat(string instructions = "");
17+
1018
/// <summary>
1119
/// Generate a new chat message
1220
/// </summary>
@@ -20,9 +28,14 @@ public Task<string> GenerateMessageAsync(
2028
CancellationToken cancellationToken = default);
2129

2230
/// <summary>
23-
/// Create a new empty chat instance
31+
/// Generate a new chat message
2432
/// </summary>
25-
/// <param name="instructions">Optional chat instructions for the AI service</param>
26-
/// <returns>Chat object</returns>
27-
public ChatHistory CreateNewChat(string instructions = "");
33+
/// <param name="chat">Chat history</param>
34+
/// <param name="requestSettings">AI request settings</param>
35+
/// <param name="cancellationToken">Async cancellation token</param>
36+
/// <returns>Stream the generated chat message in string format</returns>
37+
public IAsyncEnumerable<string> GenerateMessageStreamAsync(
38+
ChatHistory chat,
39+
ChatRequestSettings? requestSettings = null,
40+
CancellationToken cancellationToken = default);
2841
}

‎samples/dotnet/kernel-syntax-examples/Example16_CustomLLM.cs‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,15 +19,18 @@
1919
*/
2020
public class MyTextCompletionService : ITextCompletion
2121
{
22-
public Task<string> CompleteAsync(
22+
public async Task<string> CompleteAsync(
2323
string text,
2424
CompleteRequestSettings requestSettings,
2525
CancellationToken cancellationToken = default)
2626
{
2727
// Your model logic here
2828
var result = "...output from your custom model...";
2929

30-
return Task.FromResult(result);
30+
// Forcing a 2 sec delay (Simulating custom LLM lag)
31+
await Task.Delay(2000, cancellationToken);
32+
33+
return result;
3134
}
3235

3336
public async IAsyncEnumerable<string> CompleteStreamAsync(

0 commit comments

Comments
 (0)