Skip to content

Commit 37ce872

Browse files
baywetCopilotCopilot
authored
.NET: Store created external input messages (#8606)
* fix(dotnet): preserve canonical external input messages Co-authored-by: Copilot <[email protected]> Copilot-Session: d7d465e5-9825-45e9-a772-123a18ff3241 * .NET: Preserve external input message shape Keep caller content ordering while applying canonical provider references and retain typed message tables for empty captures. Co-authored-by: Copilot <[email protected]> * Fix ambiguous canonical content matching Co-authored-by: Copilot <[email protected]> Copilot-Session: 325f7b3f-1997-44b5-9c3d-cf1d4d277840 * Handle multiple canonicalized attachments Co-authored-by: baywet <[email protected]> * chore: linting --------- Co-authored-by: Copilot <[email protected]> Co-authored-by: copilot-swe-agent[bot] <[email protected]> Co-authored-by: baywet <[email protected]> Copilot-Session: d7d465e5-9825-45e9-a772-123a18ff3241 Copilot-Session: 325f7b3f-1997-44b5-9c3d-cf1d4d277840
1 parent 1370903 commit 37ce872

4 files changed

Lines changed: 253 additions & 26 deletions

File tree

‎dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Extensions/ChatMessageExtensions.cs‎

Lines changed: 76 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -31,10 +31,9 @@ public static RecordValue ToRecord(this ChatMessage message) =>
3131
/// Strategy: keep <paramref name="inputMessage"/> as the base — it has the server-generated
3232
/// <see cref="ChatMessage.MessageId"/> and any provider-augmented metadata, and is forward-
3333
/// compatible with new properties added on <see cref="ChatMessage"/> in the abstractions
34-
/// layer. Only the <see cref="ChatMessage.Contents"/> list is mutated to substitute
35-
/// original <see cref="TextContent"/> items in place (and append any extras the round-trip
36-
/// dropped). Non-text content items returned by the service are left untouched so
37-
/// server-side references survive.
34+
/// layer. Only the <see cref="ChatMessage.Contents"/> list is mutated to preserve the
35+
/// caller's ordering while substituting server-side references using stable identity first,
36+
/// then the provider-preserved order when all remaining media items correspond one-to-one.
3837
/// </para>
3938
/// </remarks>
4039
public static ChatMessage MergeForLastMessage(this ChatMessage input, ChatMessage? inputMessage)
@@ -44,32 +43,93 @@ public static ChatMessage MergeForLastMessage(this ChatMessage input, ChatMessag
4443
return input;
4544
}
4645

47-
// Build a queue of the original text items, in order. Fall back to ChatMessage.Text
48-
// if the input has no explicit TextContent entries.
49-
Queue<TextContent> originalTexts = new(input.Contents.OfType<TextContent>());
50-
if (originalTexts.Count == 0 && !string.IsNullOrEmpty(input.Text))
46+
List<AIContent> inputNonTextContents = [.. input.Contents.Where(static content => content is not TextContent)];
47+
List<AIContent> canonicalNonTextContents = [.. inputMessage.Contents.Where(static content => content is not TextContent)];
48+
AIContent[] replacements = [.. inputNonTextContents];
49+
bool[] inputContentMatched = new bool[inputNonTextContents.Count];
50+
bool[] canonicalContentUsed = new bool[canonicalNonTextContents.Count];
51+
52+
for (int inputIndex = 0; inputIndex < inputNonTextContents.Count; inputIndex++)
5153
{
52-
originalTexts.Enqueue(new TextContent(input.Text));
54+
for (int canonicalIndex = 0; canonicalIndex < canonicalNonTextContents.Count; canonicalIndex++)
55+
{
56+
if (!canonicalContentUsed[canonicalIndex] &&
57+
HasSameStableIdentity(inputNonTextContents[inputIndex], canonicalNonTextContents[canonicalIndex]))
58+
{
59+
replacements[inputIndex] = canonicalNonTextContents[canonicalIndex];
60+
inputContentMatched[inputIndex] = true;
61+
canonicalContentUsed[canonicalIndex] = true;
62+
break;
63+
}
64+
}
5365
}
5466

55-
// Replace TextContent items in inputMessage.Contents with the originals, in order.
56-
for (int i = 0; i < inputMessage.Contents.Count && originalTexts.Count > 0; i++)
67+
List<int> unmatchedInputIndexes = [.. Enumerable.Range(0, inputNonTextContents.Count).Where(index => !inputContentMatched[index])];
68+
List<int> unmatchedCanonicalIndexes = [.. Enumerable.Range(0, canonicalNonTextContents.Count).Where(index => !canonicalContentUsed[index])];
69+
70+
if (unmatchedInputIndexes.Count == unmatchedCanonicalIndexes.Count &&
71+
unmatchedInputIndexes.All(index => IsMediaContent(inputNonTextContents[index])) &&
72+
unmatchedCanonicalIndexes.All(index => IsMediaContent(canonicalNonTextContents[index])))
5773
{
58-
if (inputMessage.Contents[i] is TextContent)
74+
for (int index = 0; index < unmatchedInputIndexes.Count; index++)
5975
{
60-
inputMessage.Contents[i] = originalTexts.Dequeue();
76+
replacements[unmatchedInputIndexes[index]] = canonicalNonTextContents[unmatchedCanonicalIndexes[index]];
6177
}
6278
}
6379

64-
// Append any remaining original text items that the round-trip dropped entirely.
65-
while (originalTexts.Count > 0)
80+
List<AIContent> mergedContents = [];
81+
int nonTextIndex = 0;
82+
83+
foreach (AIContent content in input.Contents)
84+
{
85+
mergedContents.Add(
86+
content is TextContent
87+
? content
88+
: replacements[nonTextIndex++]);
89+
}
90+
91+
if (mergedContents.Count == 0 && !string.IsNullOrEmpty(input.Text))
6692
{
67-
inputMessage.Contents.Add(originalTexts.Dequeue());
93+
mergedContents.Add(new TextContent(input.Text));
94+
}
95+
96+
if (mergedContents.Count == 0)
97+
{
98+
return inputMessage;
99+
}
100+
101+
if (inputNonTextContents.Count == 0)
102+
{
103+
mergedContents.AddRange(canonicalNonTextContents);
104+
}
105+
106+
inputMessage.Contents.Clear();
107+
foreach (AIContent content in mergedContents)
108+
{
109+
inputMessage.Contents.Add(content);
68110
}
69111

70112
return inputMessage;
71113
}
72114

115+
private static bool HasSameStableIdentity(AIContent input, AIContent canonical) =>
116+
ReferenceEquals(input, canonical) ||
117+
(input, canonical) switch
118+
{
119+
(HostedFileContent inputFile, HostedFileContent canonicalFile) =>
120+
string.Equals(inputFile.FileId, canonicalFile.FileId, StringComparison.Ordinal),
121+
(UriContent inputUri, UriContent canonicalUri) =>
122+
inputUri.Uri == canonicalUri.Uri &&
123+
string.Equals(inputUri.MediaType, canonicalUri.MediaType, StringComparison.OrdinalIgnoreCase),
124+
(DataContent inputData, DataContent canonicalData) =>
125+
string.Equals(inputData.Uri, canonicalData.Uri, StringComparison.Ordinal) &&
126+
string.Equals(inputData.MediaType, canonicalData.MediaType, StringComparison.OrdinalIgnoreCase),
127+
_ => false
128+
};
129+
130+
private static bool IsMediaContent(AIContent content) =>
131+
content is DataContent or UriContent or HostedFileContent;
132+
73133
public static TableValue ToTable(this IEnumerable<ChatMessage> messages) =>
74134
FormulaValue.NewTable(TypeSchema.Message.RecordType, messages.Select(message => message.ToRecord()));
75135

‎dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/ObjectModel/RequestExternalInputExecutor.cs‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
// Copyright (c) Microsoft. All rights reserved.
22

3+
using System.Collections.Generic;
34
using System.Linq;
45
using System.Threading;
56
using System.Threading.Tasks;
@@ -37,20 +38,24 @@ public static class Steps
3738

3839
public async ValueTask CaptureResponseAsync(IWorkflowContext context, ExternalInputResponse response, CancellationToken cancellationToken)
3940
{
41+
IEnumerable<ChatMessage> capturedMessages = response.Messages;
4042
string? workflowConversationId = context.GetWorkflowConversation();
4143
if (workflowConversationId is not null)
4244
{
45+
List<ChatMessage> canonicalMessages = [];
4346
foreach (ChatMessage inputMessage in response.Messages)
4447
{
45-
await agentProvider.CreateMessageAsync(workflowConversationId, inputMessage, cancellationToken).ConfigureAwait(false);
48+
ChatMessage canonicalMessage = await agentProvider.CreateMessageAsync(workflowConversationId, inputMessage, cancellationToken).ConfigureAwait(false);
49+
canonicalMessages.Add(inputMessage.MergeForLastMessage(canonicalMessage));
4650
}
51+
capturedMessages = canonicalMessages;
4752
}
48-
ChatMessage? lastMessage = response.Messages.LastOrDefault();
53+
ChatMessage? lastMessage = capturedMessages.LastOrDefault();
4954
if (lastMessage is not null)
5055
{
5156
await context.SetLastMessageAsync(lastMessage).ConfigureAwait(false);
5257
}
53-
await this.AssignAsync(this.Model.Variable?.Path, response.Messages.ToFormula(), context).ConfigureAwait(false);
58+
await this.AssignAsync(this.Model.Variable?.Path, capturedMessages.ToTable(), context).ConfigureAwait(false);
5459

5560
await context.RaiseCompletionEventAsync(this.Model, cancellationToken).ConfigureAwait(false);
5661
}

‎dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.UnitTests/Extensions/ChatMessageExtensionsTests.cs‎

Lines changed: 67 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,7 @@ public void ToTableWithEmptyMessages()
101101
// Assert
102102
Assert.NotNull(result);
103103
Assert.Empty(result.Rows);
104+
Assert.Equal(TypeSchema.Message.RecordType.ToTable(), result.Type);
104105
}
105106

106107
[Fact]
@@ -834,7 +835,7 @@ public void MergeForLastMessageReplacesTextInPlaceAndKeepsServerMedia()
834835
}
835836

836837
[Fact]
837-
public void MergeForLastMessageAppendsOriginalTextWhenRoundTripHasNoTextSlot()
838+
public void MergeForLastMessagePreservesOriginalOrderWhenRoundTripHasNoTextSlot()
838839
{
839840
// Arrange: round-tripped message has only media (no text slot to replace).
840841
HostedFileContent serverRef = new("file-1");
@@ -844,10 +845,69 @@ public void MergeForLastMessageAppendsOriginalTextWhenRoundTripHasNoTextSlot()
844845
// Act
845846
ChatMessage result = input.MergeForLastMessage(roundTripped);
846847

847-
// Assert: media kept; original text appended at end.
848+
// Assert: original order is kept while media is replaced by the server reference.
848849
Assert.Collection(result.Contents,
849-
c => Assert.Same(serverRef, c),
850-
c => Assert.Equal("middle", Assert.IsType<TextContent>(c).Text));
850+
c => Assert.Equal("middle", Assert.IsType<TextContent>(c).Text),
851+
c => Assert.Same(serverRef, c));
852+
}
853+
854+
[Fact]
855+
public void MergeForLastMessagePreservesOriginalMediaWhenCanonicalCountDiffers()
856+
{
857+
// Arrange
858+
DataContent firstInput = new("data:image/jpeg;base64,QUE=", "image/jpeg");
859+
DataContent secondInput = new("data:image/jpeg;base64,QkI=", "image/jpeg");
860+
HostedFileContent canonical = new("file-b");
861+
ChatMessage input = new(ChatRole.User, [firstInput, secondInput]);
862+
ChatMessage roundTripped = new(ChatRole.User, [canonical]) { MessageId = "id" };
863+
864+
// Act
865+
ChatMessage result = input.MergeForLastMessage(roundTripped);
866+
867+
// Assert
868+
Assert.Collection(result.Contents,
869+
content => Assert.Same(firstInput, content),
870+
content => Assert.Same(secondInput, content));
871+
}
872+
873+
[Fact]
874+
public void MergeForLastMessageMapsCanonicalizedMediaByProviderOrder()
875+
{
876+
// Arrange
877+
DataContent firstInput = new("data:image/jpeg;base64,QUE=", "image/jpeg");
878+
DataContent secondInput = new("data:image/jpeg;base64,QkI=", "image/jpeg");
879+
HostedFileContent firstCanonical = new("file-a");
880+
HostedFileContent secondCanonical = new("file-b");
881+
ChatMessage input = new(ChatRole.User, [firstInput, secondInput]);
882+
ChatMessage roundTripped = new(ChatRole.User, [firstCanonical, secondCanonical]) { MessageId = "id" };
883+
884+
// Act
885+
ChatMessage result = input.MergeForLastMessage(roundTripped);
886+
887+
// Assert
888+
Assert.Collection(result.Contents,
889+
content => Assert.Same(firstCanonical, content),
890+
content => Assert.Same(secondCanonical, content));
891+
}
892+
893+
[Fact]
894+
public void MergeForLastMessageMatchesReorderedCanonicalMediaByStableIdentity()
895+
{
896+
// Arrange
897+
HostedFileContent firstInput = new("file-a");
898+
HostedFileContent secondInput = new("file-b");
899+
HostedFileContent firstCanonical = new("file-a");
900+
HostedFileContent secondCanonical = new("file-b");
901+
ChatMessage input = new(ChatRole.User, [firstInput, secondInput]);
902+
ChatMessage roundTripped = new(ChatRole.User, [secondCanonical, firstCanonical]) { MessageId = "id" };
903+
904+
// Act
905+
ChatMessage result = input.MergeForLastMessage(roundTripped);
906+
907+
// Assert
908+
Assert.Collection(result.Contents,
909+
content => Assert.Same(firstCanonical, content),
910+
content => Assert.Same(secondCanonical, content));
851911
}
852912

853913
[Fact]
@@ -864,10 +924,10 @@ public void MergeForLastMessageReplacesMultipleTextSlotsInOrder()
864924

865925
// Assert
866926
Assert.Collection(result.Contents,
867-
c => Assert.Same(firstRef, c),
868927
c => Assert.Equal("first", Assert.IsType<TextContent>(c).Text),
869-
c => Assert.Same(secondRef, c),
870-
c => Assert.Equal("second", Assert.IsType<TextContent>(c).Text));
928+
c => Assert.Equal("second", Assert.IsType<TextContent>(c).Text),
929+
c => Assert.Same(firstRef, c),
930+
c => Assert.Same(secondRef, c));
871931
}
872932

873933
[Fact]

‎dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.UnitTests/ObjectModel/RequestExternalInputExecutorTest.cs‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,85 @@ await this.CaptureResponseTestAsync(
8585
expectMessagesCreated: true);
8686
}
8787

88+
[Fact]
89+
public async Task CaptureResponseUsesCanonicalMessagesWithWorkflowConversationAsync()
90+
{
91+
// Arrange
92+
const string VariableName = "TestVariable";
93+
this.State.Set(SystemScope.Names.ConversationId, FormulaValue.New("WorkflowConversationId"), VariableScopeNames.System);
94+
95+
RequestExternalInput model = this.CreateModel(nameof(CaptureResponseUsesCanonicalMessagesWithWorkflowConversationAsync), VariableName);
96+
MockAgentProvider mockAgentProvider = new();
97+
RequestExternalInputExecutor action = new(model, mockAgentProvider.Object, this.State);
98+
99+
ChatMessage[] inputMessages =
100+
[
101+
new(ChatRole.User, [new TextContent("First message"), new HostedFileContent("caller-file-1")]) { MessageId = "caller-message-1" },
102+
new(ChatRole.User, [new TextContent("Second message"), new HostedFileContent("caller-file-2")]) { MessageId = "caller-message-2" },
103+
];
104+
ChatMessage[] canonicalMessages =
105+
[
106+
new(ChatRole.User, [new TextContent("Canonical first"), new HostedFileContent("provider-file-1")]) { MessageId = "provider-message-1" },
107+
new(ChatRole.User, [new TextContent("Canonical second"), new HostedFileContent("provider-file-2")]) { MessageId = "provider-message-2" },
108+
];
109+
int canonicalMessageIndex = 0;
110+
mockAgentProvider
111+
.Setup(p => p.CreateMessageAsync("WorkflowConversationId", It.IsAny<ChatMessage>(), It.IsAny<CancellationToken>()))
112+
.Returns(() => Task.FromResult(canonicalMessages[canonicalMessageIndex++]));
113+
114+
ExternalInputResponse response = new(inputMessages);
115+
116+
// Act
117+
WorkflowEvent[] events =
118+
await this.ExecuteAsync(
119+
RequestExternalInputExecutor.Steps.Capture(action.Id),
120+
(context, message, cancellationToken) => action.CaptureResponseAsync(context, response, cancellationToken));
121+
122+
// Assert
123+
VerifyCompletionEvent(events);
124+
ChatMessage[] expectedMessages =
125+
[
126+
new(ChatRole.User, [new TextContent("First message"), new HostedFileContent("provider-file-1")]) { MessageId = "provider-message-1" },
127+
new(ChatRole.User, [new TextContent("Second message"), new HostedFileContent("provider-file-2")]) { MessageId = "provider-message-2" },
128+
];
129+
this.VerifyState(VariableName, expectedMessages.ToTable());
130+
this.VerifyState(SystemScope.Names.LastMessage, VariableScopeNames.System, expectedMessages[1].ToRecord());
131+
this.VerifyState(SystemScope.Names.LastMessageId, VariableScopeNames.System, FormulaValue.New("provider-message-2"));
132+
this.VerifyState(SystemScope.Names.LastMessageText, VariableScopeNames.System, FormulaValue.New("Second message"));
133+
}
134+
135+
[Fact]
136+
public async Task CaptureResponseUsesCallerMessagesWithoutWorkflowConversationAsync()
137+
{
138+
// Arrange
139+
const string VariableName = "TestVariable";
140+
RequestExternalInput model = this.CreateModel(nameof(CaptureResponseUsesCallerMessagesWithoutWorkflowConversationAsync), VariableName);
141+
MockAgentProvider mockAgentProvider = new();
142+
RequestExternalInputExecutor action = new(model, mockAgentProvider.Object, this.State);
143+
ChatMessage[] inputMessages =
144+
[
145+
new(ChatRole.User, [new TextContent("First message"), new HostedFileContent("caller-file-1")]) { MessageId = "caller-message-1" },
146+
new(ChatRole.User, [new TextContent("Second message"), new HostedFileContent("caller-file-2")]) { MessageId = "caller-message-2" },
147+
];
148+
ExternalInputResponse response = new(inputMessages);
149+
150+
// Act
151+
WorkflowEvent[] events =
152+
await this.ExecuteAsync(
153+
RequestExternalInputExecutor.Steps.Capture(action.Id),
154+
(context, message, cancellationToken) => action.CaptureResponseAsync(context, response, cancellationToken));
155+
156+
// Assert
157+
VerifyCompletionEvent(events);
158+
mockAgentProvider.Verify(
159+
p => p.CreateMessageAsync(It.IsAny<string>(), It.IsAny<ChatMessage>(), It.IsAny<CancellationToken>()),
160+
Times.Never);
161+
this.VerifyState(VariableName, inputMessages.ToTable());
162+
this.VerifyState(SystemScope.Names.LastMessage, VariableScopeNames.System, inputMessages[1].ToRecord());
163+
this.VerifyState(SystemScope.Names.LastMessageId, VariableScopeNames.System, FormulaValue.New("caller-message-2"));
164+
this.VerifyState(SystemScope.Names.LastMessageText, VariableScopeNames.System, FormulaValue.New("Second message"));
165+
}
166+
88167
[Fact]
89168
public async Task CaptureResponseWithEmptyMessagesAsync()
90169
{
@@ -94,6 +173,29 @@ await this.CaptureResponseTestAsync(
94173
messageCount: 0);
95174
}
96175

176+
[Fact]
177+
public async Task CaptureResponseWithEmptyMessagesPreservesMessageTableTypeAsync()
178+
{
179+
// Arrange
180+
const string VariableName = "TestVariable";
181+
RequestExternalInput model = this.CreateModel(nameof(CaptureResponseWithEmptyMessagesPreservesMessageTableTypeAsync), VariableName);
182+
MockAgentProvider mockAgentProvider = new();
183+
RequestExternalInputExecutor action = new(model, mockAgentProvider.Object, this.State);
184+
ExternalInputResponse response = new([]);
185+
186+
// Act
187+
WorkflowEvent[] events =
188+
await this.ExecuteAsync(
189+
RequestExternalInputExecutor.Steps.Capture(action.Id),
190+
(context, message, cancellationToken) => action.CaptureResponseAsync(context, response, cancellationToken));
191+
192+
// Assert
193+
VerifyCompletionEvent(events);
194+
TableValue table = Assert.IsAssignableFrom<TableValue>(this.State.Get(VariableName));
195+
Assert.Empty(table.Rows);
196+
Assert.Equal(TypeSchema.Message.RecordType.ToTable(), table.Type);
197+
}
198+
97199
[Fact]
98200
public async Task CaptureResponseWithEmptyMessagesAndWorkflowConversationAsync()
99201
{

0 commit comments

Comments
 (0)