Ref struct in scanned assembly results in app crash at startup, if sagas are present (#7195)

* Fix ref struct scan for sagas (#7187)

* Prevent saga conventions from throwing on ref struct

* Better test

* Tweaks

---------

Co-authored-by: Mike Minutillo <[email protected]>

* Handle netframework

* Use fx agnostic way

---------

Co-authored-by: Mike Minutillo <[email protected]>

Author: andreasohlund

src/NServiceBus.AcceptanceTests/Core/Conventions/When_scanning_an_assembly_containing_a_ref_struct_and_sagas_enabled.cs

@@ -0,0 +1,53 @@
+namespace NServiceBus.AcceptanceTests.Core.Conventions;
+
+using System;
+using System.Threading.Tasks;
+using AcceptanceTesting;
+using EndpointTemplates;
+using NServiceBus.AcceptanceTesting.Customization;
+using NUnit.Framework;
+
+public class When_scanning_an_assembly_containing_a_ref_struct_and_sagas_enabled : NServiceBusAcceptanceTest
+{
+    [Test]
+    public void It_should_not_throw_an_exception()
+        => Assert.DoesNotThrowAsync(
+            () => Scenario.Define<ScenarioContext>()
+                          .WithEndpoint<EndpointWithASaga>()
+                          .Run()
+    );
+
+    // HINT: This will get picked up by the AssemblyRouteSource created by the routing call below
+    // Even though it is not a message type, it is still checked by passing it to conventions.
+    // The conventions added by Sagas were throwing an exception when passed a ref struct.
+    // See https://github.com/Particular/NServiceBus/issues/7179 for details.
+    ref struct RefStruct { }
+
+    class EndpointWithASaga : EndpointConfigurationBuilder
+    {
+        public EndpointWithASaga() => EndpointSetup<DefaultServer>(cfg => cfg
+            .ConfigureRouting()
+            .RouteToEndpoint(
+                typeof(RefStruct).Assembly,
+                Conventions.EndpointNamingConvention(typeof(EndpointWithASaga))
+            )
+        );
+
+        class RealSagaToSetUpConventions : Saga<RealSagaToSetUpConventions.RealSagaToSetUpConventionsSagaData>, IAmStartedByMessages<SomeMessage>
+        {
+            public Task Handle(SomeMessage message, IMessageHandlerContext context) => Task.CompletedTask;
+            protected override void ConfigureHowToFindSaga(SagaPropertyMapper<RealSagaToSetUpConventionsSagaData> mapper)
+                => mapper.MapSaga(saga => saga.BusinessId).ToMessage<SomeMessage>(msg => msg.BusinessId);
+
+            public class RealSagaToSetUpConventionsSagaData : ContainSagaData
+            {
+                public virtual Guid BusinessId { get; set; }
+            }
+        }
+    }
+
+    public class SomeMessage : IMessage
+    {
+        public Guid BusinessId { get; set; }
+    }
+}

src/NServiceBus.Core/Sagas/Sagas.cs

@@ -3,6 +3,8 @@
     using System;
     using System.Collections.Generic;
     using System.Linq;
+    using System.Reflection;
+    using System.Runtime.CompilerServices;
     using Microsoft.Extensions.DependencyInjection;
     using NServiceBus.Sagas;
 
@@ -104,6 +106,13 @@ static bool IsCompatible(Type t, Type source)
 
         static bool IsTypeATimeoutHandledByAnySaga(Type type, IEnumerable<Type> sagas)
         {
+            // MakeGenericType() throws an exception if passed a ref struct type
+            // Messages cannot be ref struct types
+            if (type.GetCustomAttribute<IsByRefLikeAttribute>() != null)
+            {
+                return false;
+            }
+
             var timeoutHandler = typeof(IHandleTimeouts<>).MakeGenericType(type);
             var messageHandler = typeof(IHandleMessages<>).MakeGenericType(type);