feat: intelligent test failure diagnosis and categorization (#4953)

* feat: add intelligent test failure categorization

Automatically categorize test failures to help users quickly understand
what went wrong. Categories include: Assertion, Timeout, NullReference,
Setup (Before hooks), Teardown (After hooks), Infrastructure (IO/network),
and Unknown. The category label is prefixed in the failure explanation
visible in test reports.

* fix: use categorizer result consistently and unwrap AggregateException at top of GetFailureStateProperty

The method was computing a category via FailureCategorizer but then
ignoring it for routing, using separate type-checking logic against the
original (potentially wrapped) exception. This caused two bugs:

1. AggregateException wrapping e.g. TaskCanceledException would be
   correctly categorized as Timeout by the categorizer, but the method's
   own `e is TaskCanceledException` check would fail since `e` was still
   the AggregateException.

2. The assertion check used a different string comparison
   (InvariantCulture) than the categorizer (Ordinal) and only checked
   for "Assertion" rather than also checking "Assert".

Now AggregateException is unwrapped once at the top, the unwrapped
exception is passed to the categorizer, and the FailureCategory enum
drives all routing and is used for constructing state properties.

Author: thomhurst

TUnit.Engine/Enums/FailureCategory.cs

@@ -0,0 +1,42 @@
+namespace TUnit.Engine.Enums;
+
+/// <summary>
+/// Categorizes test failures to help users quickly understand what went wrong.
+/// </summary>
+internal enum FailureCategory
+{
+    /// <summary>
+    /// TUnit assertion failure (AssertionException).
+    /// </summary>
+    Assertion,
+
+    /// <summary>
+    /// Timeout or cancellation failure (OperationCanceledException, TaskCanceledException, TimeoutException).
+    /// </summary>
+    Timeout,
+
+    /// <summary>
+    /// NullReferenceException in user code.
+    /// </summary>
+    NullReference,
+
+    /// <summary>
+    /// Failure in a Before/BeforeEvery hook.
+    /// </summary>
+    Setup,
+
+    /// <summary>
+    /// Failure in an After/AfterEvery hook.
+    /// </summary>
+    Teardown,
+
+    /// <summary>
+    /// File, network, or other I/O exception.
+    /// </summary>
+    Infrastructure,
+
+    /// <summary>
+    /// Unrecognized failure type.
+    /// </summary>
+    Unknown
+}

TUnit.Engine/Services/FailureCategorizer.cs

@@ -0,0 +1,92 @@
+using TUnit.Core.Exceptions;
+using TUnit.Engine.Enums;
+
+namespace TUnit.Engine.Services;
+
+/// <summary>
+/// Examines exceptions from test failures and categorizes them to help users
+/// quickly understand what went wrong.
+/// </summary>
+internal static class FailureCategorizer
+{
+    /// <summary>
+    /// Categorizes the given exception into a <see cref="FailureCategory"/>.
+    /// Unwraps <see cref="AggregateException"/> to inspect the first inner exception.
+    /// </summary>
+    public static FailureCategory Categorize(Exception exception)
+    {
+        // Unwrap AggregateException to get the real cause
+        var ex = exception is AggregateException { InnerExceptions.Count: > 0 } agg
+            ? agg.InnerExceptions[0]
+            : exception;
+
+        // Setup hooks (Before*)
+        if (ex is BeforeTestException
+            or BeforeClassException
+            or BeforeAssemblyException
+            or BeforeTestSessionException
+            or BeforeTestDiscoveryException)
+        {
+            return FailureCategory.Setup;
+        }
+
+        // Teardown hooks (After*)
+        if (ex is AfterTestException
+            or AfterClassException
+            or AfterAssemblyException
+            or AfterTestSessionException
+            or AfterTestDiscoveryException)
+        {
+            return FailureCategory.Teardown;
+        }
+
+        // Assertion failures - check by type name to support third-party assertion libraries
+        if (ex.GetType().Name.Contains("Assertion", StringComparison.Ordinal)
+            || ex.GetType().Name.Contains("Assert", StringComparison.Ordinal))
+        {
+            return FailureCategory.Assertion;
+        }
+
+        // Timeout / cancellation
+        if (ex is OperationCanceledException
+            or TaskCanceledException
+            or System.TimeoutException
+            or TUnit.Core.Exceptions.TimeoutException)
+        {
+            return FailureCategory.Timeout;
+        }
+
+        // NullReference
+        if (ex is NullReferenceException)
+        {
+            return FailureCategory.NullReference;
+        }
+
+        // Infrastructure (I/O, network, file system)
+        if (ex is IOException
+            or System.Net.Sockets.SocketException
+            or System.Net.Http.HttpRequestException
+            or UnauthorizedAccessException)
+        {
+            return FailureCategory.Infrastructure;
+        }
+
+        return FailureCategory.Unknown;
+    }
+
+    /// <summary>
+    /// Returns a short human-readable label for the category,
+    /// suitable for prefixing failure messages in reports.
+    /// </summary>
+    public static string GetLabel(FailureCategory category) => category switch
+    {
+        FailureCategory.Assertion => "Assertion Failure",
+        FailureCategory.Timeout => "Timeout",
+        FailureCategory.NullReference => "Null Reference",
+        FailureCategory.Setup => "Setup Failure",
+        FailureCategory.Teardown => "Teardown Failure",
+        FailureCategory.Infrastructure => "Infrastructure Failure",
+        FailureCategory.Unknown => "Test Failure",
+        _ => "Test Failure"
+    };
+}

TUnit.Engine/TUnitMessageBus.cs

@@ -6,6 +6,7 @@
 using Microsoft.Testing.Platform.TestHost;
 using TUnit.Core;
 using TUnit.Engine.CommandLineProviders;
+using TUnit.Engine.Enums;
 using TUnit.Engine.Exceptions;
 using TUnit.Engine.Extensions;
 using TUnit.Engine.Services;
@@ -143,19 +144,27 @@ public ValueTask PublishOutputUpdate(TestNode testNode)
 
     private static TestNodeStateProperty GetFailureStateProperty(TestContext testContext, Exception e, TimeSpan duration)
     {
-        if (testContext.Metadata.TestDetails.Timeout != null
-            && e is TaskCanceledException or OperationCanceledException or TimeoutException
+        // Unwrap AggregateException once so all downstream logic sees the real cause
+        var unwrapped = e is AggregateException { InnerExceptions.Count: > 0 } agg
+            ? agg.InnerExceptions[0]
+            : e;
+
+        var category = FailureCategorizer.Categorize(unwrapped);
+        var categoryLabel = FailureCategorizer.GetLabel(category);
+
+        if (category == FailureCategory.Timeout
+            && testContext.Metadata.TestDetails.Timeout != null
             && duration >= testContext.Metadata.TestDetails.Timeout.Value)
         {
-            return new TimeoutTestNodeStateProperty($"Test timed out after {testContext.Metadata.TestDetails.Timeout.Value.TotalMilliseconds}ms");
+            return new TimeoutTestNodeStateProperty($"[{categoryLabel}] Test timed out after {testContext.Metadata.TestDetails.Timeout.Value.TotalMilliseconds}ms");
         }
 
-        if (e.GetType().Name.Contains("Assertion", StringComparison.InvariantCulture))
+        if (category == FailureCategory.Assertion)
         {
-            return new FailedTestNodeStateProperty(e);
+            return new FailedTestNodeStateProperty(unwrapped, $"[{categoryLabel}] {unwrapped.Message}");
         }
 
-        return new ErrorTestNodeStateProperty(e);
+        return new ErrorTestNodeStateProperty(unwrapped, $"[{categoryLabel}] {unwrapped.Message}");
     }
 
     public Task<bool> IsEnabledAsync()