Skip to content

Commit 2063930

Browse files
thomhurstthomhurst
committed
feat: add performance regression testing with baseline comparison
Add [PerformanceBaseline(MaxDurationMs = N)] attribute for tracking test execution times and detecting performance regressions. By default, violations emit warnings; use --performance-baseline-fail to turn them into test failures. Add --report-timing / --report-timing-path to write a JSON timing report containing per-test duration data for external comparison.
1 parent 808e8ac commit 2063930

5 files changed

Lines changed: 449 additions & 0 deletions

File tree

‎TUnit.Core/Attributes/TestMetadata/PerformanceBaselineAttribute.cs‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
using TUnit.Core.Interfaces;
2+
3+
namespace TUnit.Core;
4+
5+
/// <summary>
6+
/// Attribute that specifies a performance baseline for a test.
7+
/// When the test exceeds the specified maximum duration, a warning is emitted.
8+
/// If <c>--performance-baseline-fail</c> is enabled, the test will be marked as failed instead.
9+
/// </summary>
10+
/// <remarks>
11+
/// <para>
12+
/// Use this attribute to track test execution times and detect performance regressions.
13+
/// The attribute measures the actual test body execution time (from TestStart to TestEnd).
14+
/// </para>
15+
///
16+
/// <para>
17+
/// The attribute can be applied at different levels:
18+
/// </para>
19+
/// <list type="bullet">
20+
/// <item>Method level: Sets baseline for a specific test method</item>
21+
/// <item>Class level: Sets baseline for all test methods in the class</item>
22+
/// <item>Assembly level: Sets baseline for all test methods in the assembly</item>
23+
/// </list>
24+
/// </remarks>
25+
/// <example>
26+
/// <code>
27+
/// // Set a 500ms performance baseline
28+
/// [Test, PerformanceBaseline(MaxDurationMs = 500)]
29+
/// public async Task FastOperation()
30+
/// {
31+
/// await DoWork(); // Warning if this takes longer than 500ms
32+
/// }
33+
/// </code>
34+
/// </example>
35+
[AttributeUsage(AttributeTargets.Method | AttributeTargets.Class | AttributeTargets.Assembly)]
36+
public class PerformanceBaselineAttribute : TUnitAttribute, ITestEndEventReceiver
37+
{
38+
/// <summary>
39+
/// When set to <c>true</c>, performance baseline violations will cause the test to fail
40+
/// instead of emitting a warning. This is set by the engine when <c>--performance-baseline-fail</c> is specified.
41+
/// </summary>
42+
internal static bool FailOnViolation { get; set; }
43+
44+
/// <summary>
45+
/// Gets or sets the maximum expected duration in milliseconds.
46+
/// If the test exceeds this duration, a warning or failure is produced.
47+
/// </summary>
48+
public int MaxDurationMs { get; set; }
49+
50+
/// <inheritdoc />
51+
#if NET
52+
public int Order => int.MaxValue; // Run after all other end receivers
53+
#else
54+
public int Order { get; } = int.MaxValue; // Run after all other end receivers
55+
#endif
56+
57+
/// <inheritdoc />
58+
public ValueTask OnTestEnd(TestContext context)
59+
{
60+
var result = context.Execution.Result;
61+
62+
if (result is null)
63+
{
64+
return default;
65+
}
66+
67+
// Only check passing tests - no point flagging a failed test for performance
68+
if (result.State != TestState.Passed)
69+
{
70+
return default;
71+
}
72+
73+
var duration = result.Duration;
74+
75+
if (duration is null)
76+
{
77+
return default;
78+
}
79+
80+
var maxDuration = TimeSpan.FromMilliseconds(MaxDurationMs);
81+
82+
if (duration.Value <= maxDuration)
83+
{
84+
return default;
85+
}
86+
87+
var message = $"Performance baseline exceeded: test took {duration.Value.TotalMilliseconds:F1}ms but baseline is {MaxDurationMs}ms";
88+
89+
if (FailOnViolation)
90+
{
91+
context.Execution.OverrideResult(TestState.Failed, message);
92+
}
93+
else
94+
{
95+
// Emit as a warning via test output
96+
context.Output.WriteError($"[PERF WARNING] {message}");
97+
}
98+
99+
return default;
100+
}
101+
}

‎TUnit.Engine/CommandLineProviders/PerformanceBaselineCommandProvider.cs‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
using Microsoft.Testing.Platform.CommandLine;
2+
using Microsoft.Testing.Platform.Extensions;
3+
using Microsoft.Testing.Platform.Extensions.CommandLine;
4+
5+
namespace TUnit.Engine.CommandLineProviders;
6+
7+
internal class PerformanceBaselineCommandProvider(IExtension extension) : ICommandLineOptionsProvider
8+
{
9+
public const string PerformanceBaselineFail = "performance-baseline-fail";
10+
11+
public Task<bool> IsEnabledAsync()
12+
{
13+
return extension.IsEnabledAsync();
14+
}
15+
16+
public string Uid => extension.Uid;
17+
18+
public string Version => extension.Version;
19+
20+
public string DisplayName => extension.DisplayName;
21+
22+
public string Description => extension.Description;
23+
24+
public IReadOnlyCollection<CommandLineOption> GetCommandLineOptions()
25+
{
26+
return
27+
[
28+
new CommandLineOption(PerformanceBaselineFail, "Fail tests that exceed their [PerformanceBaseline] max duration instead of warning", ArgumentArity.Zero, false)
29+
];
30+
}
31+
32+
public Task<ValidationResult> ValidateOptionArgumentsAsync(CommandLineOption commandOption, string[] arguments)
33+
{
34+
return ValidationResult.ValidTask;
35+
}
36+
37+
public Task<ValidationResult> ValidateCommandLineOptionsAsync(ICommandLineOptions commandLineOptions)
38+
{
39+
return ValidationResult.ValidTask;
40+
}
41+
}

‎TUnit.Engine/CommandLineProviders/TimingReporterCommandProvider.cs‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
using Microsoft.Testing.Platform.CommandLine;
2+
using Microsoft.Testing.Platform.Extensions;
3+
using Microsoft.Testing.Platform.Extensions.CommandLine;
4+
5+
namespace TUnit.Engine.CommandLineProviders;
6+
7+
internal class TimingReporterCommandProvider(IExtension extension) : ICommandLineOptionsProvider
8+
{
9+
public const string ReportTiming = "report-timing";
10+
public const string ReportTimingOutputPath = "report-timing-path";
11+
12+
public Task<bool> IsEnabledAsync()
13+
{
14+
return extension.IsEnabledAsync();
15+
}
16+
17+
public string Uid => extension.Uid;
18+
19+
public string Version => extension.Version;
20+
21+
public string DisplayName => extension.DisplayName;
22+
23+
public string Description => extension.Description;
24+
25+
public IReadOnlyCollection<CommandLineOption> GetCommandLineOptions()
26+
{
27+
return
28+
[
29+
new CommandLineOption(ReportTiming, "Write test timing data to a JSON file for performance comparison", ArgumentArity.Zero, false),
30+
new CommandLineOption(ReportTimingOutputPath, "Path to output timing JSON file (default: TestResults/timing-report.json)", ArgumentArity.ExactlyOne, false)
31+
];
32+
}
33+
34+
public Task<ValidationResult> ValidateOptionArgumentsAsync(CommandLineOption commandOption, string[] arguments)
35+
{
36+
if (commandOption.Name == ReportTimingOutputPath && arguments.Length != 1)
37+
{
38+
return ValidationResult.InvalidTask("A single output path must be specified for --report-timing-path");
39+
}
40+
41+
return ValidationResult.ValidTask;
42+
}
43+
44+
public Task<ValidationResult> ValidateCommandLineOptionsAsync(ICommandLineOptions commandLineOptions)
45+
{
46+
return ValidationResult.ValidTask;
47+
}
48+
}

‎TUnit.Engine/Extensions/TestApplicationBuilderExtensions.cs‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
using Microsoft.Testing.Platform.Capabilities.TestFramework;
33
using Microsoft.Testing.Platform.Helpers;
44
using Microsoft.Testing.Platform.Services;
5+
using TUnit.Core;
56
using TUnit.Engine.Capabilities;
67
using TUnit.Engine.CommandLineProviders;
78
using TUnit.Engine.Framework;
@@ -24,6 +25,9 @@ public static void AddTUnit(this ITestApplicationBuilder testApplicationBuilder)
2425
var junitReporter = new JUnitReporter(extension);
2526
var junitReporterCommandProvider = new JUnitReporterCommandProvider(extension);
2627

28+
var timingReporter = new TimingReporter(extension);
29+
var timingReporterCommandProvider = new TimingReporterCommandProvider(extension);
30+
2731
testApplicationBuilder.RegisterTestFramework(
2832
serviceProvider => new TestFrameworkCapabilities(CreateCapabilities(serviceProvider)),
2933
(capabilities, serviceProvider) => new TUnitTestFramework(extension, serviceProvider, capabilities));
@@ -52,6 +56,10 @@ public static void AddTUnit(this ITestApplicationBuilder testApplicationBuilder)
5256
// JUnit reporter configuration
5357
testApplicationBuilder.CommandLine.AddProvider(() => junitReporterCommandProvider);
5458

59+
// Performance baseline and timing reporter command providers
60+
testApplicationBuilder.CommandLine.AddProvider(() => new PerformanceBaselineCommandProvider(extension));
61+
testApplicationBuilder.CommandLine.AddProvider(() => timingReporterCommandProvider);
62+
5563
testApplicationBuilder.TestHost.AddDataConsumer(serviceProvider =>
5664
{
5765
// Apply command-line configuration if provided
@@ -76,6 +84,34 @@ public static void AddTUnit(this ITestApplicationBuilder testApplicationBuilder)
7684
return junitReporter;
7785
});
7886
testApplicationBuilder.TestHost.AddTestHostApplicationLifetime(_ => junitReporter);
87+
88+
// Timing reporter configuration (enabled via --report-timing)
89+
testApplicationBuilder.TestHost.AddDataConsumer(serviceProvider =>
90+
{
91+
var commandLineOptions = serviceProvider.GetRequiredService<ICommandLineOptions>();
92+
93+
// Configure performance baseline fail mode
94+
if (commandLineOptions.IsOptionSet(PerformanceBaselineCommandProvider.PerformanceBaselineFail))
95+
{
96+
PerformanceBaselineAttribute.FailOnViolation = true;
97+
}
98+
99+
// Enable timing reporter when --report-timing is specified
100+
if (commandLineOptions.IsOptionSet(TimingReporterCommandProvider.ReportTiming))
101+
{
102+
timingReporter.Enable();
103+
}
104+
105+
// Configure timing reporter output path
106+
if (commandLineOptions.TryGetOptionArgumentList(TimingReporterCommandProvider.ReportTimingOutputPath, out var pathArgs))
107+
{
108+
timingReporter.SetOutputPath(pathArgs[0]);
109+
timingReporter.Enable(); // Specifying a path implies enabling
110+
}
111+
112+
return timingReporter;
113+
});
114+
testApplicationBuilder.TestHost.AddTestHostApplicationLifetime(_ => timingReporter);
79115
}
80116

81117
private static IReadOnlyCollection<ITestFrameworkCapability> CreateCapabilities(IServiceProvider serviceProvider)

0 commit comments

Comments
 (0)