Skip to content

Commit bbb8152

Browse files
authored
.Net: Add AI suggested tweaks to prompts and use IChatClient (#12038)
### Motivation and Context - #10100 - #10712 ### Description - Add AI suggested tweaks to prompts - Make a few more prompts configurable - Change Behavior to use IChatClient ### Contribution Checklist <!-- Before submitting this PR, please make sure: --> - [ ] The code builds clean without any errors or warnings - [ ] The PR follows the [SK Contribution Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md) and the [pre-submission formatting script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts) raises no violations - [ ] All unit tests pass, and I have added new tests where possible - [ ] I didn't break anyone 😄
1 parent 7ab834e commit bbb8152

11 files changed

Lines changed: 244 additions & 143 deletions

File tree

‎dotnet/samples/Concepts/Agents/ChatCompletion_Whiteboard.cs‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
// Copyright (c) Microsoft. All rights reserved.
22

3+
using Azure.AI.OpenAI;
4+
using Azure.Identity;
5+
using Microsoft.Extensions.AI;
36
using Microsoft.SemanticKernel;
47
using Microsoft.SemanticKernel.Agents;
58
using Microsoft.SemanticKernel.ChatCompletion;
@@ -27,11 +30,15 @@ public class ChatCompletion_Whiteboard(ITestOutputHelper output) : BaseTest(outp
2730
[Fact]
2831
private async Task UseWhiteboardAsync()
2932
{
30-
// Create our agent and add our finance plugin with auto function invocation.
31-
Kernel kernel = this.CreateKernelWithChatCompletion();
33+
var chatClient = new AzureOpenAIClient(new Uri(TestConfiguration.AzureOpenAI.Endpoint), new AzureCliCredential())
34+
.GetChatClient(TestConfiguration.AzureOpenAI.ChatDeploymentName)
35+
.AsIChatClient();
3236

3337
// Create the whiteboard.
34-
var whiteboardBehavior = new WhiteboardBehavior(kernel.Clone());
38+
var whiteboardBehavior = new WhiteboardBehavior(chatClient);
39+
40+
// Create our agent and add our finance plugin with auto function invocation.
41+
Kernel kernel = this.CreateKernelWithChatCompletion();
3542

3643
// Create the agent with our sample plugin.
3744
kernel.Plugins.AddFromType<VMPlugin>();

‎dotnet/src/IntegrationTests/IntegrationTests.csproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@
4848
<PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" />
4949
<PackageReference Include="Microsoft.Extensions.Configuration.Json" />
5050
<PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" />
51+
<PackageReference Include="Microsoft.Extensions.AI.OpenAI" />
5152
<PackageReference Include="Microsoft.Extensions.Http" />
5253
<PackageReference Include="Microsoft.Extensions.Http.Resilience" />
5354
<PackageReference Include="Microsoft.Extensions.TimeProvider.Testing" />

‎dotnet/src/IntegrationTests/Memory/WhiteboardBehaviorTests.cs‎

Lines changed: 87 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,12 @@
44
using System.Collections.Generic;
55
using System.Linq;
66
using System.Threading.Tasks;
7+
using Azure.AI.OpenAI;
78
using Azure.Identity;
89
using Microsoft.Extensions.AI;
910
using Microsoft.Extensions.Configuration;
10-
using Microsoft.SemanticKernel;
1111
using Microsoft.SemanticKernel.Memory;
1212
using SemanticKernel.IntegrationTests.TestSettings;
13-
using xRetry;
1413
using Xunit;
1514
using Xunit.Abstractions;
1615

@@ -29,67 +28,35 @@ public class WhiteboardBehaviorTests
2928
.Build();
3029

3130
private readonly ITestOutputHelper _output;
32-
private readonly Kernel _kernel;
31+
private readonly IChatClient _chatClient;
3332

3433
public WhiteboardBehaviorTests(ITestOutputHelper output)
3534
{
36-
AzureOpenAIConfiguration configuration = this._configuration.GetSection("AzureOpenAI").Get<AzureOpenAIConfiguration>()!;
37-
38-
var kernelBuilder = Kernel.CreateBuilder();
39-
kernelBuilder.AddAzureOpenAIChatCompletion(
40-
deploymentName: configuration.ChatDeploymentName!,
41-
endpoint: configuration.Endpoint,
42-
credentials: new AzureCliCredential());
43-
this._kernel = kernelBuilder.Build();
4435
this._output = output;
45-
}
46-
47-
[RetryTheory]
48-
[MemberData(nameof(AddMessagesToWhiteboardData))]
49-
public async Task CanAddMessagesToWhiteboardAsync(ChatMessage[] chatMessages, string[][] expectedWhiteboardContent)
50-
{
51-
// Arrange
52-
var whiteboardBehavior = new WhiteboardBehavior(this._kernel);
53-
54-
// Act
55-
foreach (var chatMessage in chatMessages)
56-
{
57-
await whiteboardBehavior.OnNewMessageAsync(null, chatMessage);
58-
}
59-
60-
// Assert
61-
await whiteboardBehavior.WhenProcessingCompleteAsync();
62-
var whiteboardContent = await whiteboardBehavior.OnModelInvokeAsync(new List<ChatMessage> { new(ChatRole.User, string.Empty) });
63-
this._output.WriteLine(string.Join(Environment.NewLine, whiteboardContent));
64-
65-
var whiteboardLines = whiteboardContent.Split('\n');
66-
foreach (var expectedContent in expectedWhiteboardContent)
67-
{
68-
bool foundAllInOneLine = false;
69-
foreach (var line in whiteboardLines)
70-
{
71-
bool foundAll = expectedContent.All(line.Contains);
72-
foundAllInOneLine = foundAllInOneLine || foundAll;
73-
}
7436

75-
Assert.True(foundAllInOneLine, $"Expected content '{string.Join(", ", expectedContent)}' not found in whiteboard content.");
76-
}
37+
AzureOpenAIConfiguration configuration = this._configuration.GetSection("AzureOpenAI").Get<AzureOpenAIConfiguration>()!;
38+
this._chatClient = new AzureOpenAIClient(new Uri(configuration.Endpoint), new AzureCliCredential())
39+
.GetChatClient(configuration.ChatDeploymentName)
40+
.AsIChatClient();
7741
}
7842

79-
public static IEnumerable<object[]> AddMessagesToWhiteboardData()
43+
[Fact]
44+
public Task AddsRequirementToWhiteboardAsync()
8045
{
81-
yield return new object[]
82-
{
46+
return this.CanAddMessagesToWhiteboardAsync(
8347
new[]
8448
{
8549
new ChatMessage(ChatRole.User, "Hello") { AuthorName = "Siobhan" },
8650
new ChatMessage(ChatRole.Assistant, "Hello, how can I help you?") { AuthorName = "Copilot" },
8751
new ChatMessage(ChatRole.User, "I want to create a presentation") { AuthorName = "Siobhan" },
8852
},
89-
new string[][] { new string[] { "REQUIREMENT", "presentation" } }
90-
};
91-
yield return new object[]
92-
{
53+
new string[][] { new string[] { "REQUIREMENT", "presentation" } });
54+
}
55+
56+
[Fact]
57+
public Task AddsRequirementsAndProposalToWhiteboardAsync()
58+
{
59+
return this.CanAddMessagesToWhiteboardAsync(
9360
new[]
9461
{
9562
new ChatMessage(ChatRole.User, "I want to create a presentation") { AuthorName = "Siobhan" },
@@ -104,10 +71,13 @@ public static IEnumerable<object[]> AddMessagesToWhiteboardData()
10471
new string[] { "REQUIREMENT", "presentation" },
10572
new string[] { "REQUIREMENT", "customers" },
10673
new string[] { "PROPOSAL", "Contoso", "Northwind" }
107-
}
108-
};
109-
yield return new object[]
110-
{
74+
});
75+
}
76+
77+
[Fact]
78+
public Task AddsDecisionToWhiteboardAsync()
79+
{
80+
return this.CanAddMessagesToWhiteboardAsync(
11181
new[]
11282
{
11383
new ChatMessage(ChatRole.User, "I want to create a presentation") { AuthorName = "Siobhan" },
@@ -121,20 +91,75 @@ public static IEnumerable<object[]> AddMessagesToWhiteboardData()
12191
new string[][]
12292
{
12393
new string[] { "DECISION", "presentation", "Contoso", "professional" }
124-
}
125-
};
126-
yield return new object[]
127-
{
94+
});
95+
}
96+
97+
[Fact]
98+
public Task AddsProposalToWhiteboardAsync()
99+
{
100+
return this.CanAddMessagesToWhiteboardAsync(
101+
new[]
102+
{
103+
new ChatMessage(ChatRole.User, "I am looking to create a VM") { AuthorName = "Siobhan" },
104+
new ChatMessage(ChatRole.User, "It should be in Europe") { AuthorName = "Siobhan" },
105+
new ChatMessage(ChatRole.User, "It should have 16GB or RAM") { AuthorName = "Siobhan" },
106+
new ChatMessage(ChatRole.User, "It should have 4 cores") { AuthorName = "Siobhan" },
107+
new ChatMessage(ChatRole.Assistant, "OK, shall I create a VM for you in Europe with 16GB of RAM, 4 cores and with the name `VM-Europe`?") { AuthorName = "Copilot" },
108+
},
109+
new string[][]
110+
{
111+
new string[] { "PROPOSAL", "Europe", "16GB", "4", "VM", "VM-Europe" }
112+
});
113+
}
114+
115+
[Fact]
116+
public Task AddsActionToWhiteboardAsync()
117+
{
118+
return this.CanAddMessagesToWhiteboardAsync(
128119
new[]
129120
{
130-
new ChatMessage(ChatRole.User, "I am looking to buy a lamp") { AuthorName = "Siobhan" },
131-
new ChatMessage(ChatRole.User, "I want it to be energy efficient") { AuthorName = "Siobhan" },
132-
new ChatMessage(ChatRole.User, "It should fit my red colour scheme.") { AuthorName = "Siobhan" },
121+
new ChatMessage(ChatRole.User, "I am looking to create a VM") { AuthorName = "Siobhan" },
122+
new ChatMessage(ChatRole.Assistant, "I need you to give me the required location, amount of RAM you need, and number of cores required.") { AuthorName = "Copilot" },
123+
new ChatMessage(ChatRole.User, "It should be in Europe") { AuthorName = "Siobhan" },
124+
new ChatMessage(ChatRole.User, "It should have 16GB or RAM") { AuthorName = "Siobhan" },
125+
new ChatMessage(ChatRole.User, "It should have 4 cores") { AuthorName = "Siobhan" },
126+
new ChatMessage(ChatRole.Assistant, "OK, shall I create a VM for you in Europe with 16GB of RAM, 4 cores and with the name `VM-Europe`?") { AuthorName = "Copilot" },
127+
new ChatMessage(ChatRole.User, "Yes, please go ahead and create that.") { AuthorName = "Siobhan" },
128+
new ChatMessage(ChatRole.Assistant, "OK, I've created the VM for you.") { AuthorName = "Copilot" },
133129
},
134130
new string[][]
135131
{
136-
new string[] { "REQUIREMENT", "lamp" }, new string[] { "REQUIREMENT", "red" }, new string[] { "REQUIREMENT", "efficient" }
132+
new string[] { "ACTION", "Europe", "16GB", "4", "VM" }
133+
});
134+
}
135+
136+
private async Task CanAddMessagesToWhiteboardAsync(ChatMessage[] chatMessages, string[][] expectedWhiteboardContent)
137+
{
138+
// Arrange
139+
var whiteboardBehavior = new WhiteboardBehavior(this._chatClient);
140+
141+
// Act
142+
foreach (var chatMessage in chatMessages)
143+
{
144+
await whiteboardBehavior.OnNewMessageAsync(null, chatMessage);
145+
}
146+
147+
// Assert
148+
await whiteboardBehavior.WhenProcessingCompleteAsync();
149+
var whiteboardContent = await whiteboardBehavior.OnModelInvokeAsync(new List<ChatMessage> { new(ChatRole.User, string.Empty) });
150+
this._output.WriteLine(string.Join(Environment.NewLine, whiteboardContent));
151+
152+
var whiteboardLines = whiteboardContent.Split('\n');
153+
foreach (var expectedContent in expectedWhiteboardContent)
154+
{
155+
bool foundAllInOneLine = false;
156+
foreach (var line in whiteboardLines)
157+
{
158+
bool foundAll = expectedContent.All(line.Contains);
159+
foundAllInOneLine = foundAllInOneLine || foundAll;
137160
}
138-
};
161+
162+
Assert.True(foundAllInOneLine, $"Expected content '{string.Join(", ", expectedContent)}' not found in whiteboard content.");
163+
}
139164
}
140165
}

‎dotnet/src/SemanticKernel.Core/Data/TextSearchBehavior/TextSearchBehavior.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ public sealed class TextSearchBehavior : AIContextBehavior
2121
{
2222
private const string DefaultPluginSearchFunctionName = "Search";
2323
private const string DefaultPluginSearchFunctionDescription = "Allows searching for additional information to help answer the user question.";
24-
private const string DefaultContextPrompt = "Consider the following information from source documents when responding to the user:";
24+
private const string DefaultContextPrompt = "## Additional Context\nConsider the following information from source documents when responding to the user:";
2525
private const string DefaultIncludeCitationsPrompt = "Include citations to the source document with document name and link if document name and link is available.";
2626

2727
private readonly ITextSearch _textSearch;

‎dotnet/src/SemanticKernel.Core/Data/TextSearchBehavior/TextSearchBehaviorOptions.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ public int Top
7070
/// to those chunks, in order to instruct the model to include citations.
7171
/// </summary>
7272
/// <value>
73-
/// Defaults to &quot;Include citations to the source document with document name and link if document name and link is available.:&quot;
73+
/// Defaults to &quot;## Additional Context\nInclude citations to the source document with document name and link if document name and link is available.:&quot;
7474
/// </value>
7575
public string? IncludeCitationsPrompt { get; init; }
7676

‎dotnet/src/SemanticKernel.Core/Memory/Mem0/Mem0Behavior.cs‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ namespace Microsoft.SemanticKernel.Memory;
3737
[Experimental("SKEXP0130")]
3838
public sealed class Mem0Behavior : AIContextBehavior
3939
{
40-
private const string DefaultContextPrompt = "Consider the following memories when answering user questions:";
40+
private const string DefaultContextPrompt = "## Memories\nConsider the following memories when answering user questions:";
4141

4242
private readonly string? _applicationId;
4343
private readonly string? _agentId;
@@ -148,7 +148,11 @@ public override async Task<string> OnModelInvokeAsync(ICollection<ChatMessage> n
148148
inputText).ConfigureAwait(false);
149149

150150
var lineSeparatedMemories = string.Join(Environment.NewLine, memories);
151-
return string.Join(Environment.NewLine, this._contextPrompt, lineSeparatedMemories);
151+
return
152+
$"""
153+
{this._contextPrompt}
154+
{lineSeparatedMemories}
155+
""";
152156
}
153157

154158
/// <summary>

‎dotnet/src/SemanticKernel.Core/Memory/Mem0/Mem0BehaviorOptions.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ public sealed class Mem0BehaviorOptions
5959
/// to those memories, in order to provide some context to the model.
6060
/// </summary>
6161
/// <value>
62-
/// Defaults to &quot;Consider the following memories when answering user questions:&quot;
62+
/// Defaults to &quot;## Memories\nConsider the following memories when answering user questions:&quot;
6363
/// </value>
6464
public string? ContextPrompt { get; init; }
6565
}

0 commit comments

Comments
 (0)