Skip to content

Commit aaf7aa6

Browse files
westey-mwestey-m
authored
.Net: Add additional integration tests for the common agent invoke api (#11186)
### Description - Add a plugin invoke integration test for the new common api. - Add invoke streaming integration tests. ### Contribution Checklist <!-- Before submitting this PR, please make sure: --> - [ ] The code builds clean without any errors or warnings - [ ] The PR follows the [SK Contribution Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md) and the [pre-submission formatting script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts) raises no violations - [ ] All unit tests pass, and I have added new tests where possible - [ ] I didn't break anyone 😄
1 parent 225a95c commit aaf7aa6

12 files changed

+283
-12
lines changed

‎dotnet/src/IntegrationTests/Agents/CommonInterfaceConformance/AgentFixture.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ namespace SemanticKernel.IntegrationTests.Agents.CommonInterfaceConformance;
1313
/// </summary>
1414
public abstract class AgentFixture : IAsyncLifetime
1515
{
16-
public abstract Agent Agent { get; }
16+
public abstract KernelAgent Agent { get; }
1717

1818
public abstract AgentThread AgentThread { get; }
1919

‎dotnet/src/IntegrationTests/Agents/CommonInterfaceConformance/AzureAIAgentFixture.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ public class AzureAIAgentFixture : AgentFixture
3030
private AzureAIAgentThread? _serviceFailingAgentThread;
3131
private AzureAIAgentThread? _createdServiceFailingAgentThread;
3232

33-
public override Agent Agent => this._agent!;
33+
public override KernelAgent Agent => this._agent!;
3434

3535
public override AgentThread AgentThread => this._thread!;
3636

‎dotnet/src/IntegrationTests/Agents/CommonInterfaceConformance/BedrockAgentFixture.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ internal sealed class BedrockAgentFixture : AgentFixture, IAsyncDisposable
3737
private readonly AmazonBedrockAgentClient _client = new();
3838
private readonly AmazonBedrockAgentRuntimeClient _runtimeClient = new();
3939

40-
public override Microsoft.SemanticKernel.Agents.Agent Agent => this._agent!;
40+
public override Microsoft.SemanticKernel.Agents.KernelAgent Agent => this._agent!;
4141

4242
public override AgentThread AgentThread => this._thread!;
4343

‎dotnet/src/IntegrationTests/Agents/CommonInterfaceConformance/ChatCompletionAgentFixture.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ public class ChatCompletionAgentFixture : AgentFixture
2626
private ChatHistoryAgentThread? _thread;
2727
private ChatHistoryAgentThread? _createdThread;
2828

29-
public override Agent Agent => this._agent!;
29+
public override KernelAgent Agent => this._agent!;
3030

3131
public override AgentThread AgentThread => this._thread!;
3232

‎dotnet/src/IntegrationTests/Agents/CommonInterfaceConformance/InvokeConformance/BedrockAgentInvokeTests.cs

+6
Original file line numberDiff line numberDiff line change
@@ -39,4 +39,10 @@ public override Task InvokeWithoutThreadCreatesThreadAsync()
3939
{
4040
return base.InvokeWithoutThreadCreatesThreadAsync();
4141
}
42+
43+
[Fact(Skip = "This test is for manual verification.")]
44+
public override Task MultiStepInvokeWithPluginAndArgOverridesAsync()
45+
{
46+
return base.MultiStepInvokeWithPluginAndArgOverridesAsync();
47+
}
4248
}

‎dotnet/src/IntegrationTests/Agents/CommonInterfaceConformance/InvokeConformance/InvokeTests.cs

+62-7
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
using Microsoft.SemanticKernel;
77
using Microsoft.SemanticKernel.Agents;
88
using Microsoft.SemanticKernel.ChatCompletion;
9+
using xRetry;
910
using Xunit;
1011

1112
namespace SemanticKernel.IntegrationTests.Agents.CommonInterfaceConformance.InvokeConformance;
@@ -22,51 +23,63 @@ public abstract class InvokeTests(Func<AgentFixture> createAgentFixture) : IAsyn
2223

2324
protected AgentFixture Fixture => this._agentFixture;
2425

25-
[Fact]
26+
[RetryFact(3, 5000)]
2627
public virtual async Task InvokeReturnsResultAsync()
2728
{
29+
// Arrange
2830
var agent = this.Fixture.Agent;
31+
32+
// Act
2933
var asyncResults = agent.InvokeAsync(new ChatMessageContent(AuthorRole.User, "What is the capital of France."), this.Fixture.AgentThread);
3034
var results = await asyncResults.ToListAsync();
31-
Assert.Single(results);
3235

36+
// Assert
37+
Assert.Single(results);
3338
var firstResult = results.First();
39+
3440
Assert.Contains("Paris", firstResult.Message.Content);
3541
Assert.NotNull(firstResult.Thread);
3642
}
3743

38-
[Fact]
44+
[RetryFact(3, 5000)]
3945
public virtual async Task InvokeWithoutThreadCreatesThreadAsync()
4046
{
47+
// Arrange
4148
var agent = this.Fixture.Agent;
49+
50+
// Act
4251
var asyncResults = agent.InvokeAsync(new ChatMessageContent(AuthorRole.User, "What is the capital of France."));
4352
var results = await asyncResults.ToListAsync();
44-
Assert.Single(results);
4553

54+
// Assert
55+
Assert.Single(results);
4656
var firstResult = results.First();
4757
Assert.Contains("Paris", firstResult.Message.Content);
4858
Assert.NotNull(firstResult.Thread);
4959

60+
// Cleanup
5061
await this.Fixture.DeleteThread(firstResult.Thread);
5162
}
5263

53-
[Fact]
64+
[RetryFact(3, 5000)]
5465
public virtual async Task ConversationMaintainsHistoryAsync()
5566
{
67+
// Arrange
5668
var q1 = "What is the capital of France.";
5769
var q2 = "What is the capital of Austria.";
58-
5970
var agent = this.Fixture.Agent;
71+
72+
// Act
6073
var asyncResults1 = agent.InvokeAsync(new ChatMessageContent(AuthorRole.User, q1), this.Fixture.AgentThread);
6174
var result1 = await asyncResults1.FirstAsync();
6275
var asyncResults2 = agent.InvokeAsync(new ChatMessageContent(AuthorRole.User, q2), result1.Thread);
6376
var result2 = await asyncResults2.FirstAsync();
6477

78+
// Assert
6579
Assert.Contains("Paris", result1.Message.Content);
6680
Assert.Contains("Austria", result2.Message.Content);
6781

6882
var chatHistory = await this.Fixture.GetChatHistory();
69-
7083
Assert.Equal(4, chatHistory.Count);
7184
Assert.Equal(2, chatHistory.Count(x => x.Role == AuthorRole.User));
7285
Assert.Equal(2, chatHistory.Count(x => x.Role == AuthorRole.Assistant));
@@ -76,6 +89,48 @@ public virtual async Task ConversationMaintainsHistoryAsync()
7689
Assert.Contains("Vienna", chatHistory[3].Content);
7790
}
7891

92+
/// <summary>
93+
/// Verifies that the agent can invoke a plugin and respects the override
94+
/// Kernel and KernelArguments provided in the options.
95+
/// The step does multiple iterations to make sure that the agent
96+
/// also manages the chat history correctly.
97+
/// </summary>
98+
[RetryFact(3, 5000)]
99+
public virtual async Task MultiStepInvokeWithPluginAndArgOverridesAsync()
100+
{
101+
// Arrange
102+
var questionsAndAnswers = new[]
103+
{
104+
("Hello", string.Empty),
105+
("What is the special soup?", "Clam Chowder"),
106+
("What is the special drink?", "Chai Tea"),
107+
("What is the special salad?", "Cobb Salad"),
108+
("Thank you", string.Empty)
109+
};
110+
111+
var agent = this.Fixture.Agent;
112+
var kernel = agent.Kernel.Clone();
113+
kernel.Plugins.AddFromType<MenuPlugin>();
114+
115+
foreach (var questionAndAnswer in questionsAndAnswers)
116+
{
117+
// Act
118+
var asyncResults = agent.InvokeAsync(
119+
new ChatMessageContent(AuthorRole.User, questionAndAnswer.Item1),
120+
this.Fixture.AgentThread,
121+
options: new()
122+
{
123+
Kernel = kernel,
124+
KernelArguments = new KernelArguments(new PromptExecutionSettings() { FunctionChoiceBehavior = FunctionChoiceBehavior.Auto() })
125+
});
126+
127+
// Assert
128+
var result = await asyncResults.FirstAsync();
129+
Assert.NotNull(result);
130+
Assert.Contains(questionAndAnswer.Item2, result.Message.Content);
131+
}
132+
}
133+
79134
public Task InitializeAsync()
80135
{
81136
this._agentFixture = createAgentFixture();

‎dotnet/src/IntegrationTests/Agents/CommonInterfaceConformance/InvokeStreamingConformance/AzureAIAgentInvokeStreamingTests.cs

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
// Copyright (c) Microsoft. All rights reserved.
2+
3+
using Xunit;
4+
5+
namespace SemanticKernel.IntegrationTests.Agents.CommonInterfaceConformance.InvokeStreamingConformance;
6+
7+
[Collection("Sequential")]
8+
public class AzureAIAgentInvokeStreamingTests() : InvokeStreamingTests(() => new AzureAIAgentFixture())
9+
{
10+
}

‎dotnet/src/IntegrationTests/Agents/CommonInterfaceConformance/InvokeStreamingConformance/ChatCompletionAgentInvokeStreamingTests.cs

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
// Copyright (c) Microsoft. All rights reserved.
2+
3+
namespace SemanticKernel.IntegrationTests.Agents.CommonInterfaceConformance.InvokeStreamingConformance;
4+
5+
public class ChatCompletionAgentInvokeStreamingTests() : InvokeStreamingTests(() => new ChatCompletionAgentFixture())
6+
{
7+
}

‎dotnet/src/IntegrationTests/Agents/CommonInterfaceConformance/InvokeStreamingConformance/InvokeStreamingTests.cs

+149
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
// Copyright (c) Microsoft. All rights reserved.
2+
3+
using System;
4+
using System.Linq;
5+
using System.Threading.Tasks;
6+
using Microsoft.SemanticKernel;
7+
using Microsoft.SemanticKernel.Agents;
8+
using Microsoft.SemanticKernel.ChatCompletion;
9+
using xRetry;
10+
using Xunit;
11+
12+
namespace SemanticKernel.IntegrationTests.Agents.CommonInterfaceConformance.InvokeStreamingConformance;
13+
14+
/// <summary>
15+
/// Base test class for testing the <see cref="Agent.InvokeStreamingAsync(ChatMessageContent, AgentThread?, AgentInvokeOptions?, System.Threading.CancellationToken)"/> method of agents.
16+
/// Each agent type should have its own derived class.
17+
/// </summary>
18+
public abstract class InvokeStreamingTests(Func<AgentFixture> createAgentFixture) : IAsyncLifetime
19+
{
20+
#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable.
21+
private AgentFixture _agentFixture;
22+
#pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable.
23+
24+
protected AgentFixture Fixture => this._agentFixture;
25+
26+
[RetryFact(3, 10_000)]
27+
public virtual async Task InvokeStreamingAsyncReturnsResultAsync()
28+
{
29+
// Arrange
30+
var agent = this.Fixture.Agent;
31+
32+
// Act
33+
var asyncResults = agent.InvokeStreamingAsync(new ChatMessageContent(AuthorRole.User, "What is the capital of France."), this.Fixture.AgentThread);
34+
var results = await asyncResults.ToListAsync();
35+
36+
// Assert
37+
var firstResult = results.First();
38+
var resultString = string.Join(string.Empty, results.Select(x => x.Message.Content));
39+
40+
Assert.Contains("Paris", resultString);
41+
Assert.NotNull(firstResult.Thread);
42+
}
43+
44+
[RetryFact(3, 10_000)]
45+
public virtual async Task InvokeStreamingAsyncWithoutThreadCreatesThreadAsync()
46+
{
47+
// Arrange
48+
var agent = this.Fixture.Agent;
49+
50+
// Act
51+
var asyncResults = agent.InvokeStreamingAsync(new ChatMessageContent(AuthorRole.User, "What is the capital of France."));
52+
var results = await asyncResults.ToListAsync();
53+
54+
// Assert
55+
var firstResult = results.First();
56+
var resultString = string.Join(string.Empty, results.Select(x => x.Message.Content));
57+
58+
Assert.Contains("Paris", resultString);
59+
Assert.NotNull(firstResult.Thread);
60+
61+
// Cleanup
62+
await this.Fixture.DeleteThread(firstResult.Thread);
63+
}
64+
65+
[RetryFact(3, 10_000)]
66+
public virtual async Task ConversationMaintainsHistoryAsync()
67+
{
68+
// Arrange
69+
var q1 = "What is the capital of France.";
70+
var q2 = "What is the capital of Austria.";
71+
var agent = this.Fixture.Agent;
72+
73+
// Act
74+
var asyncResults1 = agent.InvokeStreamingAsync(new ChatMessageContent(AuthorRole.User, q1), this.Fixture.AgentThread);
75+
var results1 = await asyncResults1.ToListAsync();
76+
var resultString1 = string.Join(string.Empty, results1.Select(x => x.Message.Content));
77+
var result1 = results1.First();
78+
79+
var asyncResults2 = agent.InvokeStreamingAsync(new ChatMessageContent(AuthorRole.User, q2), result1.Thread);
80+
var results2 = await asyncResults2.ToListAsync();
81+
var resultString2 = string.Join(string.Empty, results2.Select(x => x.Message.Content));
82+
83+
// Assert
84+
Assert.Contains("Paris", resultString1);
85+
Assert.Contains("Austria", resultString2);
86+
87+
var chatHistory = await this.Fixture.GetChatHistory();
88+
Assert.Equal(4, chatHistory.Count);
89+
Assert.Equal(2, chatHistory.Count(x => x.Role == AuthorRole.User));
90+
Assert.Equal(2, chatHistory.Count(x => x.Role == AuthorRole.Assistant));
91+
Assert.Equal(q1, chatHistory[0].Content);
92+
Assert.Equal(q2, chatHistory[2].Content);
93+
Assert.Contains("Paris", chatHistory[1].Content);
94+
Assert.Contains("Vienna", chatHistory[3].Content);
95+
}
96+
97+
/// <summary>
98+
/// Verifies that the agent can invoke a plugin and respects the override
99+
/// Kernel and KernelArguments provided in the options.
100+
/// The step does multiple iterations to make sure that the agent
101+
/// also manages the chat history correctly.
102+
/// </summary>
103+
[RetryFact(3, 10_000)]
104+
public virtual async Task MultiStepInvokeStreamingAsyncWithPluginAndArgOverridesAsync()
105+
{
106+
// Arrange
107+
var questionsAndAnswers = new[]
108+
{
109+
("Hello", string.Empty),
110+
("What is the special soup?", "Clam Chowder"),
111+
("What is the special drink?", "Chai Tea"),
112+
("What is the special salad?", "Cobb Salad"),
113+
("Thank you", string.Empty)
114+
};
115+
116+
var agent = this.Fixture.Agent;
117+
var kernel = agent.Kernel.Clone();
118+
kernel.Plugins.AddFromType<MenuPlugin>();
119+
120+
foreach (var questionAndAnswer in questionsAndAnswers)
121+
{
122+
// Act
123+
var asyncResults = agent.InvokeStreamingAsync(
124+
new ChatMessageContent(AuthorRole.User, questionAndAnswer.Item1),
125+
this.Fixture.AgentThread,
126+
options: new()
127+
{
128+
Kernel = kernel,
129+
KernelArguments = new KernelArguments(new PromptExecutionSettings() { FunctionChoiceBehavior = FunctionChoiceBehavior.Auto() })
130+
});
131+
var results = await asyncResults.ToListAsync();
132+
133+
// Assert
134+
var resultString = string.Join(string.Empty, results.Select(x => x.Message.Content));
135+
Assert.Contains(questionAndAnswer.Item2, resultString);
136+
}
137+
}
138+
139+
public Task InitializeAsync()
140+
{
141+
this._agentFixture = createAgentFixture();
142+
return this._agentFixture.InitializeAsync();
143+
}
144+
145+
public Task DisposeAsync()
146+
{
147+
return this._agentFixture.DisposeAsync();
148+
}
149+
}

‎dotnet/src/IntegrationTests/Agents/CommonInterfaceConformance/InvokeStreamingConformance/OpenAIAssistantAgentInvokeStreamingTests.cs

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
// Copyright (c) Microsoft. All rights reserved.
2+
3+
using Xunit;
4+
5+
namespace SemanticKernel.IntegrationTests.Agents.CommonInterfaceConformance.InvokeStreamingConformance;
6+
7+
[Collection("Sequential")]
8+
public class OpenAIAssistantAgentInvokeStreamingTests() : InvokeStreamingTests(() => new OpenAIAssistantAgentFixture())
9+
{
10+
}

‎dotnet/src/IntegrationTests/Agents/CommonInterfaceConformance/MenuPlugin.cs

+34
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
// Copyright (c) Microsoft. All rights reserved.
2+
3+
using System.ComponentModel;
4+
using Microsoft.SemanticKernel;
5+
6+
namespace SemanticKernel.IntegrationTests.Agents.CommonInterfaceConformance;
7+
8+
/// <summary>
9+
/// A test plugin used to verify the ability of Semantic Kernel's Common Agent Interface to invoke plugins.
10+
/// </summary>
11+
#pragma warning disable CA1812 // Avoid uninstantiated internal classes
12+
internal sealed class MenuPlugin
13+
{
14+
[KernelFunction, Description("Provides a list of specials from the menu.")]
15+
[System.Diagnostics.CodeAnalysis.SuppressMessage("Design", "CA1024:Use properties where appropriate", Justification = "Too smart")]
16+
public string GetSpecials()
17+
{
18+
return
19+
"""
20+
Special Soup: Clam Chowder
21+
Special Salad: Cobb Salad
22+
Special Drink: Chai Tea
23+
""";
24+
}
25+
26+
[KernelFunction, Description("Provides the price of the requested menu item.")]
27+
public string GetItemPrice(
28+
[Description("The name of the menu item.")]
29+
string menuItem)
30+
{
31+
return "$9.99";
32+
}
33+
}
34+
#pragma warning restore CA1812 // Avoid uninstantiated internal classes

‎dotnet/src/IntegrationTests/Agents/CommonInterfaceConformance/OpenAIAssistantAgentFixture.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ public class OpenAIAssistantAgentFixture : AgentFixture
3434
private OpenAIAssistantAgentThread? _serviceFailingAgentThread;
3535
private OpenAIAssistantAgentThread? _createdServiceFailingAgentThread;
3636

37-
public override Agent Agent => this._agent!;
37+
public override KernelAgent Agent => this._agent!;
3838

3939
public override AgentThread AgentThread => this._thread!;
4040

0 commit comments

Comments
 (0)