Skip to content

Commit fe1839d

Browse files
committed
feat: add intelligent test failure categorization
Automatically categorize test failures to help users quickly understand what went wrong. Categories include: Assertion, Timeout, NullReference, Setup (Before hooks), Teardown (After hooks), Infrastructure (IO/network), and Unknown. The category label is prefixed in the failure explanation visible in test reports.
1 parent 512b71c commit fe1839d

3 files changed

Lines changed: 140 additions & 3 deletions

File tree

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
namespace TUnit.Engine.Enums;
2+
3+
/// <summary>
4+
/// Categorizes test failures to help users quickly understand what went wrong.
5+
/// </summary>
6+
internal enum FailureCategory
7+
{
8+
/// <summary>
9+
/// TUnit assertion failure (AssertionException).
10+
/// </summary>
11+
Assertion,
12+
13+
/// <summary>
14+
/// Timeout or cancellation failure (OperationCanceledException, TaskCanceledException, TimeoutException).
15+
/// </summary>
16+
Timeout,
17+
18+
/// <summary>
19+
/// NullReferenceException in user code.
20+
/// </summary>
21+
NullReference,
22+
23+
/// <summary>
24+
/// Failure in a Before/BeforeEvery hook.
25+
/// </summary>
26+
Setup,
27+
28+
/// <summary>
29+
/// Failure in an After/AfterEvery hook.
30+
/// </summary>
31+
Teardown,
32+
33+
/// <summary>
34+
/// File, network, or other I/O exception.
35+
/// </summary>
36+
Infrastructure,
37+
38+
/// <summary>
39+
/// Unrecognized failure type.
40+
/// </summary>
41+
Unknown
42+
}
Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
using TUnit.Core.Exceptions;
2+
using TUnit.Engine.Enums;
3+
4+
namespace TUnit.Engine.Services;
5+
6+
/// <summary>
7+
/// Examines exceptions from test failures and categorizes them to help users
8+
/// quickly understand what went wrong.
9+
/// </summary>
10+
internal static class FailureCategorizer
11+
{
12+
/// <summary>
13+
/// Categorizes the given exception into a <see cref="FailureCategory"/>.
14+
/// Unwraps <see cref="AggregateException"/> to inspect the first inner exception.
15+
/// </summary>
16+
public static FailureCategory Categorize(Exception exception)
17+
{
18+
// Unwrap AggregateException to get the real cause
19+
var ex = exception is AggregateException { InnerExceptions.Count: > 0 } agg
20+
? agg.InnerExceptions[0]
21+
: exception;
22+
23+
// Setup hooks (Before*)
24+
if (ex is BeforeTestException
25+
or BeforeClassException
26+
or BeforeAssemblyException
27+
or BeforeTestSessionException
28+
or BeforeTestDiscoveryException)
29+
{
30+
return FailureCategory.Setup;
31+
}
32+
33+
// Teardown hooks (After*)
34+
if (ex is AfterTestException
35+
or AfterClassException
36+
or AfterAssemblyException
37+
or AfterTestSessionException
38+
or AfterTestDiscoveryException)
39+
{
40+
return FailureCategory.Teardown;
41+
}
42+
43+
// Assertion failures - check by type name to support third-party assertion libraries
44+
if (ex.GetType().Name.Contains("Assertion", StringComparison.Ordinal)
45+
|| ex.GetType().Name.Contains("Assert", StringComparison.Ordinal))
46+
{
47+
return FailureCategory.Assertion;
48+
}
49+
50+
// Timeout / cancellation
51+
if (ex is OperationCanceledException
52+
or TaskCanceledException
53+
or System.TimeoutException
54+
or TUnit.Core.Exceptions.TimeoutException)
55+
{
56+
return FailureCategory.Timeout;
57+
}
58+
59+
// NullReference
60+
if (ex is NullReferenceException)
61+
{
62+
return FailureCategory.NullReference;
63+
}
64+
65+
// Infrastructure (I/O, network, file system)
66+
if (ex is IOException
67+
or System.Net.Sockets.SocketException
68+
or System.Net.Http.HttpRequestException
69+
or UnauthorizedAccessException)
70+
{
71+
return FailureCategory.Infrastructure;
72+
}
73+
74+
return FailureCategory.Unknown;
75+
}
76+
77+
/// <summary>
78+
/// Returns a short human-readable label for the category,
79+
/// suitable for prefixing failure messages in reports.
80+
/// </summary>
81+
public static string GetLabel(FailureCategory category) => category switch
82+
{
83+
FailureCategory.Assertion => "Assertion Failure",
84+
FailureCategory.Timeout => "Timeout",
85+
FailureCategory.NullReference => "Null Reference",
86+
FailureCategory.Setup => "Setup Failure",
87+
FailureCategory.Teardown => "Teardown Failure",
88+
FailureCategory.Infrastructure => "Infrastructure Failure",
89+
FailureCategory.Unknown => "Test Failure",
90+
_ => "Test Failure"
91+
};
92+
}

TUnit.Engine/TUnitMessageBus.cs

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -143,19 +143,22 @@ public ValueTask PublishOutputUpdate(TestNode testNode)
143143

144144
private static TestNodeStateProperty GetFailureStateProperty(TestContext testContext, Exception e, TimeSpan duration)
145145
{
146+
var category = FailureCategorizer.Categorize(e);
147+
var categoryLabel = FailureCategorizer.GetLabel(category);
148+
146149
if (testContext.Metadata.TestDetails.Timeout != null
147150
&& e is TaskCanceledException or OperationCanceledException or TimeoutException
148151
&& duration >= testContext.Metadata.TestDetails.Timeout.Value)
149152
{
150-
return new TimeoutTestNodeStateProperty($"Test timed out after {testContext.Metadata.TestDetails.Timeout.Value.TotalMilliseconds}ms");
153+
return new TimeoutTestNodeStateProperty($"[{categoryLabel}] Test timed out after {testContext.Metadata.TestDetails.Timeout.Value.TotalMilliseconds}ms");
151154
}
152155

153156
if (e.GetType().Name.Contains("Assertion", StringComparison.InvariantCulture))
154157
{
155-
return new FailedTestNodeStateProperty(e);
158+
return new FailedTestNodeStateProperty(e, $"[{categoryLabel}] {e.Message}");
156159
}
157160

158-
return new ErrorTestNodeStateProperty(e);
161+
return new ErrorTestNodeStateProperty(e, $"[{categoryLabel}] {e.Message}");
159162
}
160163

161164
public Task<bool> IsEnabledAsync()

0 commit comments

Comments
 (0)