Skip to content

Commit 6a3d535

Browse files
authored
.NET: Add regression tests and sample guidance for stable agent IDs in checkpointed workflows (#7415)
* Add regression tests and sample guidance for stable agent IDs in checkpointed workflows * Updated tests to address PR comments * Improve test for checkpoint state.
1 parent 73f48d2 commit 6a3d535

3 files changed

Lines changed: 258 additions & 29 deletions

File tree

‎dotnet/samples/03-workflows/Checkpoint/CheckpointAndRehydrate/Program.cs‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,14 +81,20 @@ private static async Task Main()
8181
}
8282
Console.WriteLine($"Number of checkpoints created: {checkpoints.Count}");
8383

84-
// Rehydrate a new workflow instance from a saved checkpoint and continue execution
84+
// <rehydrate_workflow>
85+
// A rehydrated workflow must preserve the topology and executor identities of the workflow that
86+
// created the checkpoint. This executor-only workflow rebuilds identically because its executors
87+
// use fixed ids. Agent-based workflows must recreate each local agent with the same
88+
// ChatClientAgentOptions.Id (and, if set, the same Name), otherwise the executor ids no longer
89+
// match the checkpoint and resume fails.
8590
var newWorkflow = WorkflowFactory.BuildWorkflow();
8691
const int CheckpointIndex = 5;
8792
Console.WriteLine($"\n\nHydrating a new workflow instance from the {CheckpointIndex + 1}th checkpoint.");
8893
CheckpointInfo savedCheckpoint = checkpoints[CheckpointIndex];
8994

9095
await using StreamingRun newCheckpointedRun =
9196
await InProcessExecution.ResumeStreamingAsync(newWorkflow, savedCheckpoint, checkpointManager);
97+
// </rehydrate_workflow>
9298

9399
await foreach (WorkflowEvent evt in newCheckpointedRun.WatchStreamAsync())
94100
{

‎dotnet/samples/03-workflows/Orchestration/Handoff/AgentRegistry.cs‎

Lines changed: 59 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -10,50 +10,81 @@
1010
/// <param name="chatClient">The <see cref="IChatClient"/> to use as the agent backend.</param>
1111
internal sealed class AgentRegistry(IChatClient chatClient)
1212
{
13+
// <stable_agent_identity>
14+
// Give each agent a stable, unique Id so its workflow executor identity stays the same when the
15+
// workflow is reconstructed (for example per request or dependency-injection scope), which keeps
16+
// checkpoints resumable. If an agent also has a Name, keep that stable too, since the executor
17+
// identity includes it. Use a fixed logical role here, not a conversation, request, or user id.
1318
internal const string IntakeAgentName = "Assistant";
14-
public AIAgent IntakeAgent { get; } = chatClient.AsAIAgent(
15-
instructions:
16-
"""
19+
public AIAgent IntakeAgent { get; } = chatClient.AsAIAgent(new ChatClientAgentOptions
20+
{
21+
Id = "intake-agent",
22+
Name = IntakeAgentName,
23+
ChatOptions = new()
24+
{
25+
Instructions =
26+
"""
1727
You receive a user request and are responsible for routing to the correct initial expert agent.
1828
""",
19-
IntakeAgentName
20-
);
29+
},
30+
});
31+
// </stable_agent_identity>
2132

2233
internal const string LiquidityAnalysisAgentName = "Liquidity Analysis";
23-
public AIAgent LiquidityAnalysisAgent { get; } = chatClient.AsAIAgent(
24-
instructions:
25-
"""
34+
public AIAgent LiquidityAnalysisAgent { get; } = chatClient.AsAIAgent(new ChatClientAgentOptions
35+
{
36+
Id = "liquidity-analysis-agent",
37+
Name = LiquidityAnalysisAgentName,
38+
ChatOptions = new()
39+
{
40+
Instructions =
41+
"""
2642
You are responsible for Liquidity Analysis.
2743
""",
28-
LiquidityAnalysisAgentName
29-
);
44+
},
45+
});
3046

3147
internal const string TaxAnalysisAgentName = "Tax Analysis";
32-
public AIAgent TaxAnalysisAgent { get; } = chatClient.AsAIAgent(
33-
instructions:
34-
"""
35-
You are responsible for Tax Analysis.
48+
public AIAgent TaxAnalysisAgent { get; } = chatClient.AsAIAgent(new ChatClientAgentOptions
49+
{
50+
Id = "tax-analysis-agent",
51+
Name = TaxAnalysisAgentName,
52+
ChatOptions = new()
53+
{
54+
Instructions =
55+
"""
56+
You are responsible for Tax Analysis.
3657
""",
37-
TaxAnalysisAgentName
38-
);
58+
},
59+
});
3960

4061
internal const string ForeignExchangeAgentName = "Foreign Exchange Analysis";
41-
public AIAgent ForeignExchangeAgent { get; } = chatClient.AsAIAgent(
42-
instructions:
43-
"""
44-
You are responsible for Foreign Exchange Analysis.
62+
public AIAgent ForeignExchangeAgent { get; } = chatClient.AsAIAgent(new ChatClientAgentOptions
63+
{
64+
Id = "foreign-exchange-agent",
65+
Name = ForeignExchangeAgentName,
66+
ChatOptions = new()
67+
{
68+
Instructions =
69+
"""
70+
You are responsible for Foreign Exchange Analysis.
4571
""",
46-
ForeignExchangeAgentName
47-
);
72+
},
73+
});
4874

4975
internal const string EquityAgentName = "Equity Analysis";
50-
public AIAgent EquityAgent { get; } = chatClient.AsAIAgent(
51-
instructions:
52-
"""
53-
You are responsible for Equity Analysis.
76+
public AIAgent EquityAgent { get; } = chatClient.AsAIAgent(new ChatClientAgentOptions
77+
{
78+
Id = "equity-analysis-agent",
79+
Name = EquityAgentName,
80+
ChatOptions = new()
81+
{
82+
Instructions =
83+
"""
84+
You are responsible for Equity Analysis.
5485
""",
55-
EquityAgentName
56-
);
86+
},
87+
});
5788

5889
public IEnumerable<AIAgent> Experts => [this.LiquidityAnalysisAgent, this.TaxAnalysisAgent, this.ForeignExchangeAgent, this.EquityAgent];
5990

Lines changed: 192 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,192 @@
1+
// Copyright (c) Microsoft. All rights reserved.
2+
3+
using System;
4+
using System.Collections.Generic;
5+
using System.IO;
6+
using System.Linq;
7+
using System.Runtime.CompilerServices;
8+
using System.Text.Json;
9+
using System.Threading;
10+
using System.Threading.Tasks;
11+
using FluentAssertions;
12+
using Microsoft.Extensions.AI;
13+
14+
namespace Microsoft.Agents.AI.Workflows.UnitTests;
15+
16+
/// <summary>
17+
/// Verifies that a workflow hosted as an <see cref="AIAgent"/> resumes a serialized session after its inner
18+
/// agents are reconstructed only when each inner agent keeps a stable executor identity. A stable
19+
/// <see cref="ChatClientAgentOptions.Id"/> is sufficient; if an agent also sets a
20+
/// <see cref="ChatClientAgentOptions.Name"/>, that name must stay stable because the executor id includes it.
21+
/// </summary>
22+
public class WorkflowAgentCheckpointIdentityTests
23+
{
24+
private const string TriageName = "triage_agent";
25+
private const string SpecialistName = "specialist_agent";
26+
private const string SpecialistReply = "SPECIALIST_REPLY";
27+
28+
// Held constant across reconstruction so resume differences come only from the inner agent identities.
29+
private const string OuterWorkflowAgentId = "workflow-agent";
30+
31+
[Fact]
32+
public async Task WorkflowAgentSession_WithStableInnerAgentIds_ResumesAcrossReconstructionAsync()
33+
{
34+
// Arrange: build a first-generation workflow agent whose inner agents have stable, explicit ids.
35+
AIAgent firstGeneration = BuildWorkflowAgent(useStableInnerIds: true);
36+
AgentSession session = await firstGeneration.CreateSessionAsync();
37+
38+
// Act: complete a first turn (triage hands off to the specialist), then serialize the session.
39+
AgentResponse firstResponse = await firstGeneration.RunAsync("Please help me.", session);
40+
firstResponse.Text.Should().Be(
41+
$"{SpecialistReply}:turn:1",
42+
"the first turn should route triage -> specialist, and the specialist observes a single user turn");
43+
44+
JsonElement serialized = await firstGeneration.SerializeSessionAsync(session);
45+
46+
// Reconstruct a completely fresh object graph (new clients, agents, and workflow) using the same stable ids,
47+
// modeling a second dependency-injection scope.
48+
AIAgent secondGeneration = BuildWorkflowAgent(useStableInnerIds: true);
49+
AgentSession resumedSession = await secondGeneration.DeserializeSessionAsync(serialized);
50+
51+
AgentResponse secondResponse = await secondGeneration.RunAsync("Anything else?", resumedSession);
52+
53+
// Assert: the specialist observes both user turns, which is only possible if the checkpointed conversation was
54+
// restored. A fresh (non-resumed) session would restart the count at turn:1, so this distinguishes a genuine
55+
// resume from a compatible-but-empty restart.
56+
secondResponse.Text.Should().Be(
57+
$"{SpecialistReply}:turn:2",
58+
"stable inner agent ids keep the executor identities compatible and the reconstructed workflow resumes from the checkpoint");
59+
}
60+
61+
[Fact]
62+
public async Task WorkflowAgentSession_WithoutStableInnerAgentIds_FailsAcrossReconstructionAsync()
63+
{
64+
// Arrange: build a first-generation workflow agent whose inner agents receive random ids (no explicit id).
65+
AIAgent firstGeneration = BuildWorkflowAgent(useStableInnerIds: false);
66+
AgentSession session = await firstGeneration.CreateSessionAsync();
67+
68+
// Complete a first turn and serialize the session; a completed handoff turn captures a checkpoint.
69+
AgentResponse firstResponse = await firstGeneration.RunAsync("Please help me.", session);
70+
firstResponse.Text.Should().Contain(SpecialistReply, "the first turn should route triage -> specialist");
71+
72+
JsonElement serialized = await firstGeneration.SerializeSessionAsync(session);
73+
74+
// Reconstruct with new random inner ids but the SAME outer workflow-agent id, proving that a stable outer id
75+
// alone does not stabilize the inner executor identities.
76+
AIAgent secondGeneration = BuildWorkflowAgent(useStableInnerIds: false);
77+
78+
// Act: deserialization itself succeeds; the incompatibility surfaces only when the resuming run validates the
79+
// checkpoint against the reconstructed workflow.
80+
AgentSession resumedSession = await secondGeneration.DeserializeSessionAsync(serialized);
81+
82+
Func<Task> resumeAndRun = () => secondGeneration.RunAsync("Anything else?", resumedSession);
83+
84+
// Assert: the second run throws because the reconstructed executor ids no longer match the checkpoint.
85+
await resumeAndRun.Should().ThrowAsync<InvalidDataException>()
86+
.WithMessage("The specified checkpoint is not compatible with the workflow associated with this runner.");
87+
}
88+
89+
[Fact]
90+
public async Task WorkflowAgentSession_WithStableIdsButChangedInnerNames_FailsAcrossReconstructionAsync()
91+
{
92+
// Arrange: first generation uses stable ids and the default inner names.
93+
AIAgent firstGeneration = BuildWorkflowAgent(useStableInnerIds: true);
94+
AgentSession session = await firstGeneration.CreateSessionAsync();
95+
96+
AgentResponse firstResponse = await firstGeneration.RunAsync("Please help me.", session);
97+
firstResponse.Text.Should().Contain(SpecialistReply, "the first turn should route triage -> specialist");
98+
99+
JsonElement serialized = await firstGeneration.SerializeSessionAsync(session);
100+
101+
// Reconstruct with the SAME stable ids but different inner names. Because the executor id is derived from
102+
// both the name and the id, changing only the name still breaks checkpoint compatibility.
103+
AIAgent secondGeneration = BuildWorkflowAgent(useStableInnerIds: true, nameSuffix: "-renamed");
104+
105+
// Act: deserialization succeeds; the incompatibility surfaces on the resuming run.
106+
AgentSession resumedSession = await secondGeneration.DeserializeSessionAsync(serialized);
107+
108+
Func<Task> resumeAndRun = () => secondGeneration.RunAsync("Anything else?", resumedSession);
109+
110+
// Assert: changing a set name invalidates the executor identity even though the id is stable.
111+
await resumeAndRun.Should().ThrowAsync<InvalidDataException>()
112+
.WithMessage("The specified checkpoint is not compatible with the workflow associated with this runner.");
113+
}
114+
115+
/// <summary>
116+
/// Builds a fresh Handoff workflow-as-agent object graph. Every call constructs new chat clients, agents, and a
117+
/// new workflow, modeling reconstruction across dependency-injection scopes.
118+
/// </summary>
119+
/// <param name="useStableInnerIds">
120+
/// When <see langword="true"/>, each inner agent is assigned a deterministic <see cref="ChatClientAgentOptions.Id"/>.
121+
/// When <see langword="false"/>, the id is left unset so each agent receives a random per-instance id.
122+
/// </param>
123+
/// <param name="nameSuffix">
124+
/// Optional suffix appended to each inner agent's <see cref="ChatClientAgentOptions.Name"/>. Used to simulate a
125+
/// reconstruction that keeps ids stable but changes names.
126+
/// </param>
127+
private static AIAgent BuildWorkflowAgent(bool useStableInnerIds, string nameSuffix = "")
128+
{
129+
AIAgent triage = CreateTriageClient().AsAIAgent(new ChatClientAgentOptions
130+
{
131+
Id = useStableInnerIds ? "triage-agent" : null,
132+
Name = TriageName + nameSuffix,
133+
Description = "Routes the request to a specialist.",
134+
ChatOptions = new() { Instructions = "Always hand off to the specialist." },
135+
});
136+
137+
AIAgent specialist = CreateSpecialistClient().AsAIAgent(new ChatClientAgentOptions
138+
{
139+
Id = useStableInnerIds ? "specialist-agent" : null,
140+
Name = SpecialistName + nameSuffix,
141+
Description = "Handles the request once triage hands off.",
142+
ChatOptions = new() { Instructions = "Answer the request." },
143+
});
144+
145+
Workflow workflow = AgentWorkflowBuilder.CreateHandoffBuilderWith(triage)
146+
.WithHandoff(triage, specialist)
147+
.Build();
148+
149+
return workflow.AsAIAgent(id: OuterWorkflowAgentId, name: OuterWorkflowAgentId);
150+
}
151+
152+
// Triage hands off to the specialist by calling the handoff tool present on the request.
153+
private static StatelessMockChatClient CreateTriageClient() => new((messages, options) =>
154+
{
155+
string handoffTool = options?.Tools?
156+
.FirstOrDefault(t => t.Name.StartsWith("handoff_to_", StringComparison.Ordinal))?.Name
157+
?? throw new InvalidOperationException("Expected a handoff tool to be available to the triage agent.");
158+
159+
return new ChatResponse(new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("handoff-call", handoffTool)]));
160+
});
161+
162+
// The specialist echoes how many user turns it has observed. Because a genuine resume restores the prior turn
163+
// from the checkpoint, the count advances across turns, distinguishing a real resume from a fresh restart.
164+
private static StatelessMockChatClient CreateSpecialistClient() => new((messages, _) =>
165+
{
166+
int observedUserTurns = messages.Count(m => m.Role == ChatRole.User);
167+
return new ChatResponse(new ChatMessage(ChatRole.Assistant, $"{SpecialistReply}:turn:{observedUserTurns}"));
168+
});
169+
170+
/// <summary>
171+
/// A minimal <see cref="IChatClient"/> whose response is a pure function of the request, so reconstructing it
172+
/// preserves behavior.
173+
/// </summary>
174+
private sealed class StatelessMockChatClient(Func<IEnumerable<ChatMessage>, ChatOptions?, ChatResponse> responseFactory) : IChatClient
175+
{
176+
public Task<ChatResponse> GetResponseAsync(IEnumerable<ChatMessage> messages, ChatOptions? options = null, CancellationToken cancellationToken = default) =>
177+
Task.FromResult(responseFactory(messages, options));
178+
179+
public async IAsyncEnumerable<ChatResponseUpdate> GetStreamingResponseAsync(
180+
IEnumerable<ChatMessage> messages, ChatOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default)
181+
{
182+
foreach (var update in (await this.GetResponseAsync(messages, options, cancellationToken).ConfigureAwait(false)).ToChatResponseUpdates())
183+
{
184+
yield return update;
185+
}
186+
}
187+
188+
public object? GetService(Type serviceType, object? serviceKey = null) => null;
189+
190+
public void Dispose() { }
191+
}
192+
}

0 commit comments

Comments
 (0)