Skip to content

Commit e91023d

Browse files
westey-mSergeyMenshykh
westey-m
and
SergeyMenshykh
authored
.Net: Add whiteboard sample (#12020)
### Motivation and Context Long running conversations can hit limits with regards to the context window size, so require truncation. This can lead to loss of context, which a whiteboarding component can help address. #10100 #10712 ### Description - Add a sample to show how to use the whiteboarding behavior. ### Contribution Checklist <!-- Before submitting this PR, please make sure: --> - [ ] The code builds clean without any errors or warnings - [ ] The PR follows the [SK Contribution Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md) and the [pre-submission formatting script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts) raises no violations - [ ] All unit tests pass, and I have added new tests where possible - [ ] I didn't break anyone 😄 --------- Co-authored-by: SergeyMenshykh <[email protected]>
1 parent ff81212 commit e91023d

4 files changed

Lines changed: 179 additions & 15 deletions

File tree

‎dotnet/samples/Concepts/Agents/ChatCompletion_Whiteboard.cs‎

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
// Copyright (c) Microsoft. All rights reserved.
2+
3+
using Microsoft.SemanticKernel;
4+
using Microsoft.SemanticKernel.Agents;
5+
using Microsoft.SemanticKernel.ChatCompletion;
6+
using Microsoft.SemanticKernel.Memory;
7+
8+
namespace Agents;
9+
10+
#pragma warning disable SKEXP0130 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
11+
12+
/// <summary>
13+
/// Demonstrate creation of <see cref="ChatCompletionAgent"/> and
14+
/// adding whiteboarding capabilities, where the most relevant information from the conversation is captured on a whiteboard.
15+
/// This is useful for long running conversations where the conversation history may need to be truncated
16+
/// over time, but you do not want to agent to lose context.
17+
/// </summary>
18+
public class ChatCompletion_Whiteboard(ITestOutputHelper output) : BaseTest(output)
19+
{
20+
private const string AgentName = "FriendlyAssistant";
21+
private const string AgentInstructions = "You are a friendly assistant";
22+
23+
/// <summary>
24+
/// Shows how to allow an agent to use a whiteboard for storing the most important information
25+
/// from a long running, truncated conversation.
26+
/// </summary>
27+
[Fact]
28+
private async Task UseWhiteboardAsync()
29+
{
30+
// Create our agent and add our finance plugin with auto function invocation.
31+
Kernel kernel = this.CreateKernelWithChatCompletion();
32+
33+
// Create the whiteboard.
34+
var whiteboardBehavior = new WhiteboardBehavior(kernel.Clone());
35+
36+
// Create the agent with our sample plugin.
37+
kernel.Plugins.AddFromType<VMPlugin>();
38+
ChatCompletionAgent agent =
39+
new()
40+
{
41+
Name = AgentName,
42+
Instructions = AgentInstructions,
43+
Kernel = kernel,
44+
Arguments = new KernelArguments(new PromptExecutionSettings { FunctionChoiceBehavior = FunctionChoiceBehavior.Auto() })
45+
};
46+
47+
// Create a chat history reducer that we can use to truncate the chat history
48+
// when it goes over 3 items.
49+
var chatHistoryReducer = new ChatHistoryTruncationReducer(3, 3);
50+
51+
// Create a thread for the agent and add the whiteboard to it.
52+
ChatHistoryAgentThread agentThread = new();
53+
agentThread.AIContextBehaviors.Add(whiteboardBehavior);
54+
55+
// Simulate a conversation with the agent.
56+
// We will also truncate the conversation once it goes over a few items.
57+
await this.InvokeWithConsoleWriteLine(agent, agentThread, whiteboardBehavior, chatHistoryReducer, "Hello");
58+
await this.InvokeWithConsoleWriteLine(agent, agentThread, whiteboardBehavior, chatHistoryReducer, "I'd like to create a VM?");
59+
await this.InvokeWithConsoleWriteLine(agent, agentThread, whiteboardBehavior, chatHistoryReducer, "I want it to have 3 cores.");
60+
await this.InvokeWithConsoleWriteLine(agent, agentThread, whiteboardBehavior, chatHistoryReducer, "I want it to have 48GB of RAM.");
61+
await this.InvokeWithConsoleWriteLine(agent, agentThread, whiteboardBehavior, chatHistoryReducer, "I want it to have a 500GB Harddrive.");
62+
await this.InvokeWithConsoleWriteLine(agent, agentThread, whiteboardBehavior, chatHistoryReducer, "I want it in Europe.");
63+
await this.InvokeWithConsoleWriteLine(agent, agentThread, whiteboardBehavior, chatHistoryReducer, "Can you make it Linux and call it 'ContosoVM'.");
64+
await this.InvokeWithConsoleWriteLine(agent, agentThread, whiteboardBehavior, chatHistoryReducer, "OK, let's call it `ContosoFinanceVM_Europe` instead.");
65+
await this.InvokeWithConsoleWriteLine(agent, agentThread, whiteboardBehavior, chatHistoryReducer, "Thanks, now I want to create another VM.");
66+
await this.InvokeWithConsoleWriteLine(agent, agentThread, whiteboardBehavior, chatHistoryReducer, "Make all the options the same as the last one, except for the region, which should be North America, and the name, which should be 'ContosoFinanceVM_NorthAmerica'.");
67+
}
68+
69+
private async Task InvokeWithConsoleWriteLine(ChatCompletionAgent agent, ChatHistoryAgentThread agentThread, WhiteboardBehavior whiteboardBehavior, IChatHistoryReducer chatHistoryReducer, string message)
70+
{
71+
// Print the user input.
72+
Console.WriteLine($"User: {message}");
73+
74+
// Invoke the agent.
75+
ChatMessageContent response = await agent.InvokeAsync(message, agentThread).FirstAsync();
76+
77+
// Print the response.
78+
Console.WriteLine($"Assistant:\n{response.Content}\n");
79+
80+
// Make sure any async whiteboard processing is complete before we print out its contents.
81+
await whiteboardBehavior.WhenProcessingCompleteAsync();
82+
83+
// Print out the whiteboard contents.
84+
Console.WriteLine("Whiteboard contents:");
85+
foreach (var item in whiteboardBehavior.CurrentWhiteboardContent)
86+
{
87+
Console.WriteLine($"- {item}");
88+
}
89+
Console.WriteLine();
90+
91+
// Truncate the chat history if it gets too big.
92+
await agentThread.ChatHistory.ReduceInPlaceAsync(chatHistoryReducer, CancellationToken.None);
93+
}
94+
95+
private sealed class VMPlugin
96+
{
97+
[KernelFunction]
98+
public Task<VMCreateResult> CreateVM(Region region, OperatingSystem os, string name, int numberOfCores, int memorySizeInGB, int hddSizeInGB)
99+
{
100+
if (name == "ContosoVM")
101+
{
102+
throw new Exception("VM name already exists");
103+
}
104+
105+
return Task.FromResult(new VMCreateResult { VMId = Guid.NewGuid().ToString() });
106+
}
107+
}
108+
109+
public class VMCreateResult
110+
{
111+
public string VMId { get; set; } = string.Empty;
112+
}
113+
114+
private enum Region
115+
{
116+
NorthAmerica,
117+
SouthAmerica,
118+
Europe,
119+
Asia,
120+
Africa,
121+
Australia
122+
}
123+
124+
private enum OperatingSystem
125+
{
126+
Windows,
127+
Linux,
128+
MacOS
129+
}
130+
}

‎dotnet/samples/Concepts/README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ dotnet test -l "console;verbosity=detailed" --filter "FullyQualifiedName=ChatCom
2525
- [OpenAIAssistant_ChartMaker](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Agents/OpenAIAssistant_ChartMaker.cs)
2626
- [ChatCompletion_Rag: Shows how to easily add RAG to an agent](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Agents/ChatCompletion_Rag.cs)
2727
- [ChatCompletion_Mem0: Shows how to add memory to an agent using mem0](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Agents/ChatCompletion_Mem0.cs)
28+
- [ChatCompletion_Whiteboard: Shows how to add short term Whiteboarding memory to an agent](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Agents/ChatCompletion_Whiteboard.cs)
2829

2930
### AudioToText - Different ways of using [`AudioToText`](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/src/SemanticKernel.Abstractions/AI/AudioToText/IAudioToTextService.cs) services to extract text from audio
3031

‎dotnet/src/IntegrationTests/Memory/WhiteboardBehaviorTests.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
using Microsoft.SemanticKernel;
1111
using Microsoft.SemanticKernel.Memory;
1212
using SemanticKernel.IntegrationTests.TestSettings;
13+
using xRetry;
1314
using Xunit;
1415
using Xunit.Abstractions;
1516

@@ -43,7 +44,7 @@ public WhiteboardBehaviorTests(ITestOutputHelper output)
4344
this._output = output;
4445
}
4546

46-
[Theory]
47+
[RetryTheory]
4748
[MemberData(nameof(AddMessagesToWhiteboardData))]
4849
public async Task CanAddMessagesToWhiteboardAsync(ChatMessage[] chatMessages, string[][] expectedWhiteboardContent)
4950
{

‎dotnet/src/SemanticKernel.Core/Memory/Whiteboard/WhiteboardBehavior.cs‎

Lines changed: 46 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,11 @@ public WhiteboardBehavior(Kernel kernel, WhiteboardBehaviorOptions? options = de
4444
/// <inheritdoc/>
4545
public override IReadOnlyCollection<AIFunction> AIFunctions => Array.Empty<AIFunction>();
4646

47+
/// <summary>
48+
/// Gets the current whiteboard content.
49+
/// </summary>
50+
public IReadOnlyList<string> CurrentWhiteboardContent => this._currentWhiteboardContent;
51+
4752
/// <inheritdoc/>
4853
public override async Task OnNewMessageAsync(string? threadId, ChatMessage newMessage, CancellationToken cancellationToken = default)
4954
{
@@ -111,9 +116,20 @@ private async Task UpdateWhiteboardAsync(ChatMessage newMessage, CancellationTok
111116
this._recentMessages.TryDequeue(out _);
112117
}
113118

114-
var inputMessagesJson = JsonSerializer.Serialize(recentMessagesList, WhiteboardBehaviorSourceGenerationContext.Default.ListChatMessage);
119+
// Extract the most important information from the input messages.
120+
var basicMessages = recentMessagesList
121+
.Select(m => new BasicMessage
122+
{
123+
AuthorName = m.AuthorName,
124+
Role = m.Role.ToString(),
125+
Text = m.Text
126+
});
127+
128+
// Serialize the input messages and the current whiteboard content to JSON.
129+
var inputMessagesJson = JsonSerializer.Serialize(basicMessages, WhiteboardBehaviorSourceGenerationContext.Default.IEnumerableBasicMessage);
115130
var currentWhiteboardJson = JsonSerializer.Serialize(this._currentWhiteboardContent, WhiteboardBehaviorSourceGenerationContext.Default.ListString);
116131

132+
// Inovke the LLM to extract the latest information from the input messages and update the whiteboard.
117133
var result = await this._kernel.InvokePromptAsync(
118134
new JsonSerializerOptions(),
119135
MaintenancePromptTemplate,
@@ -127,11 +143,12 @@ private async Task UpdateWhiteboardAsync(ChatMessage newMessage, CancellationTok
127143
},
128144
cancellationToken: cancellationToken).ConfigureAwait(false);
129145

146+
// Update the current whiteboard content with the LLM result.
130147
this._currentWhiteboardContent = JsonSerializer.Deserialize(result.ToString(), WhiteboardBehaviorSourceGenerationContext.Default.ListString) ?? [];
131148
}
132149

133150
/// <summary>
134-
/// Gets or sets the prompt template to use for extracting user facts and merging them with existing facts.
151+
/// Gets the prompt template to use for maintaining the whiteboard.
135152
/// </summary>
136153
private const string MaintenancePromptTemplate =
137154
"""
@@ -143,57 +160,59 @@ Decisions should contain all information about the decision made.
143160
When a decision is made, the proposals and requirements that led to the decision should be removed from the whiteboard.
144161
Whiteboard entries should be concise and to the point.
145162
The whiteboard can contain more than one piece of information for each category.
146-
The maximum number of messages allowed on the whiteboard at one time is {{$maxWhiteboardMessages}}. Combine messages or remove the least important messages from the list if the maximum number of messages is reached.
163+
The maximum number of messages allowed on the whiteboard at one time is {{$maxWhiteboardMessages}}.
164+
Combine messages or remove the least important messages from the list if the maximum number of messages is reached.
165+
Prioritize keeping detailed decisions for longer than requirements and proposals.
147166
Categorize whiteboard entries with a prefix of REQUIREMENT, PROPOSAL or DECISION.
148167
149168
Here are 7 few shot examples:
150169
151170
EXAMPLES START
152171
153172
New Message:
154-
[{"AuthorName":"Mary","Role":"user","Contents":[{"$type":"text","Text":"I want the colour scheme to be green and brown."}]}]
173+
[{"AuthorName":"Mary","Role":"user","Text":"I want the colour scheme to be green and brown."}]
155174
Current Whiteboard:
156175
["REQUIREMENT - Mary wants to create a presentation."]
157176
New Whiteboard:
158177
["REQUIREMENT - Mary wants to create a presentation.", "REQUIREMENT - The presentation colour schema should be green and brown."]
159178
160179
New Message:
161-
[{"AuthorName":"John","Role":"user","Contents":[{"$type":"text","Text":"I need you to help me with my homework."}]}]
180+
[{"AuthorName":"John","Role":"user","Text":"I need you to help me with my homework."}]
162181
Current Whiteboard:
163182
[]
164183
New Whiteboard:
165184
["REQUIREMENT - John wants help with homework."]
166185
167186
New Message:
168-
[{"AuthorName":"John","Role":"user","Contents":[{"$type":"text","Text":"Hello"}]}]
187+
[{"AuthorName":"John","Role":"user","Text":"Hello"}]
169188
Current Whiteboard:
170189
[]
171190
New Whiteboard:
172191
[]
173192
174193
New Message:
175-
[{"AuthorName":"Mary","Role":"user","Contents":[{"$type":"text","Text":"I've changed my mind, I want to go to London instead."}]}]
194+
[{"AuthorName":"Mary","Role":"user","Text":"I've changed my mind, I want to go to London instead."}]
176195
Current Whiteboard:
177196
["REQUIREMENT - Mary wants to book a flight.", "REQUIREMENT - The flight should be to Paris."]
178197
New Whiteboard:
179198
["REQUIREMENT - Mary wants to book a flight.", "REQUIREMENT - The flight should be to London."]
180199
181200
New Message:
182-
[{"AuthorName":"TravelAgent","Role":"assistant","Contents":[{"$type":"text","Text":"Here is an itinerary for your trip to Paris. Departing on the 17th of June at 10:00 AM and returning on the 20th of June at 5:00 PM with direct flights to Paris Charles de Gaul airport on NotsocheapoAir. The cost of the flights are EUR 243."}]}]
201+
[{"AuthorName":"TravelAgent","Role":"assistant","Text":"Here is an itinerary for your trip to Paris. Departing on the 17th of June at 10:00 AM and returning on the 20th of June at 5:00 PM with direct flights to Paris Charles de Gaul airport on NotsocheapoAir. The cost of the flights are EUR 243."}]
183202
Current Whiteboard:
184203
["REQUIREMENT - Mary wants to book a flight.", "REQUIREMENT - The flight should be to Paris during the week of 16th of June 2025."]
185204
New Whiteboard:
186205
["REQUIREMENT - Mary wants to book a flight.", "REQUIREMENT - The flight should be to Paris during the week of 16th of June 2025.", "PROPOSAL - The current proposed itinerary by the TravelAgent Assistant is to depart on the 17th of June at 10:00 AM and return on the 20th of June at 5:00 PM with direct flights to Paris Charles de Gaul airport on NotsocheapoAir. The cost of the flights are EUR 243."]
187206
188207
New Message:
189-
[{"AuthorName":"Mary","Role":"user","Contents":[{"$type":"text","Text":"That sounds good, let's book that."}]}]
208+
[{"AuthorName":"Mary","Role":"user","Text":"That sounds good, let's book that."}]
190209
Current Whiteboard:
191210
["REQUIREMENT - Mary wants to book a flight.", "REQUIREMENT - The flight should be to Paris during the week of 16th of June 2025.", "PROPOSAL - The current proposed itinerary by the TravelAgent Assistant is to depart on the 17th of June at 10:00 AM and return on the 20th of June at 5:00 PM with direct flights to Paris Charles de Gaul airport on NotsocheapoAir. The cost of the flights are EUR 243."]
192211
New Whiteboard:
193212
[""DECISION - Mary decided to book the flight departing on the 17th of June at 10:00 AM and returning on the 20th of June at 5:00 PM with direct flights to Paris Charles de Gaul airport on NotsocheapoAir. The cost of the flights are EUR 243."]
194213
195214
New Message:
196-
[{"AuthorName":"Mary","Role":"user","Contents":[{"$type":"text","Text":"I don't like the suggested option. Can I leave a day earlier and fly with anyone but NotsocheapoAir?"}]}]
215+
[{"AuthorName":"Mary","Role":"user","Text":"I don't like the suggested option. Can I leave a day earlier and fly with anyone but NotsocheapoAir?"}]
197216
Current Whiteboard:
198217
["REQUIREMENT - Mary wants to book a flight.", "REQUIREMENT - The flight should be to Paris during the week of 16th of June 2025.", "PROPOSAL - The current proposed itinerary by the TravelAgent Assistant is to depart on the 17th of June at 10:00 AM and return on the 20th of June at 5:00 PM with direct flights to Paris Charles de Gaul airport on NotsocheapoAir. The cost of the flights are EUR 243."]
199218
New Whiteboard:
@@ -209,17 +228,30 @@ EXAMPLES END
209228
{{$currentWhiteboard}}
210229
New Whiteboard:
211230
""";
231+
232+
/// <summary>
233+
/// A simple message class that contains just the most basic msessage information
234+
/// that is required to pass to the LLM.
235+
/// </summary>
236+
internal class BasicMessage
237+
{
238+
public string? AuthorName { get; set; }
239+
public string Role { get; set; } = string.Empty;
240+
public string Text { get; set; } = string.Empty;
241+
}
212242
}
213243

244+
/// <summary>
245+
/// Source generated json serializer for <see cref="WhiteboardBehavior"/>.
246+
/// </summary>
214247
[Experimental("SKEXP0130")]
215248
[JsonSourceGenerationOptions(JsonSerializerDefaults.General,
216-
UseStringEnumConverter = true,
249+
UseStringEnumConverter = false,
217250
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
218251
WriteIndented = false)]
219-
[JsonSerializable(typeof(List<ChatMessage>))]
252+
[JsonSerializable(typeof(IEnumerable<WhiteboardBehavior.BasicMessage>))]
253+
[JsonSerializable(typeof(WhiteboardBehavior.BasicMessage))]
220254
[JsonSerializable(typeof(List<string>))]
221-
[JsonSerializable(typeof(ChatMessage))]
222-
[JsonSerializable(typeof(ChatRole))]
223255
internal partial class WhiteboardBehaviorSourceGenerationContext : JsonSerializerContext
224256
{
225257
}

0 commit comments

Comments
 (0)