- Generate relative contentRoot paths in test manifests

- Fixed the public API declarations
- React to changes in the WebApplicationFactory
- Use solution-relative contentRoot path, when the path from metadata either is not available or doesn't exist.
- If none of the paths exist, use the AppContext.BaseDirectory instead.
- Fix tests with outdated expectations

Author: mkArtakMSFT

src/Identity/test/Identity.FunctionalTests/Infrastructure/ServerFactory.cs

@@ -63,7 +63,7 @@ protected override IHost CreateHost(IHostBuilder builder)
         return result;
     }
 
-    protected override TestServer CreateServer(IWebHostBuilder builder)
+    protected override ITestServer CreateServer(IWebHostBuilder builder)
     {
         var result = base.CreateServer(builder);
         EnsureDatabaseCreated(result.Host.Services);

src/Mvc/Mvc.Testing.Tasks/src/GenerateMvcTestManifestTask.cs

@@ -1,8 +1,10 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using System.Runtime.Serialization.Json;
 using System.Text;
 using Microsoft.Build.Framework;
@@ -33,11 +35,14 @@ public override bool Execute()
     {
         using var fileStream = File.Create(ManifestPath);
         var output = new Dictionary<string, string>();
+        var manifestDirectory = Path.GetDirectoryName(ManifestPath);
+
         foreach (var project in Projects)
         {
             var contentRoot = project.GetMetadata("ContentRoot");
             var assemblyName = project.GetMetadata("Identity");
-            output[assemblyName] = contentRoot;
+            var relativeContentRoot = GetRelativePath(manifestDirectory, contentRoot);
+            output[assemblyName] = relativeContentRoot;
         }
 
         var serializer = new DataContractJsonSerializer(typeof(Dictionary<string, string>), new DataContractJsonSerializerSettings
@@ -49,4 +54,44 @@ public override bool Execute()
 
         return !Log.HasLoggedErrors;
     }
+
+    private static string GetRelativePath(string? relativeTo, string path)
+    {
+        if (string.IsNullOrEmpty(relativeTo))
+        {
+            return path;
+        }
+
+        // Ensure the paths are absolute
+        string absoluteRelativeTo = Path.GetFullPath(relativeTo);
+        string absolutePath = Path.GetFullPath(path);
+
+        // Split the paths into their components
+        string[] relativeToParts = absoluteRelativeTo.TrimEnd(Path.DirectorySeparatorChar).Split(Path.DirectorySeparatorChar);
+        string[] pathParts = absolutePath.TrimEnd(Path.DirectorySeparatorChar).Split(Path.DirectorySeparatorChar);
+
+        // Find the common base path length
+        int commonLength = 0;
+        while (commonLength < relativeToParts.Length && commonLength < pathParts.Length &&
+               string.Equals(relativeToParts[commonLength], pathParts[commonLength], StringComparison.OrdinalIgnoreCase))
+        {
+            commonLength++;
+        }
+
+        // Calculate the number of directories to go up from the relativeTo path
+        int upDirectories = relativeToParts.Length - commonLength;
+
+        // Build the relative path
+        string relativePath = string.Join(Path.DirectorySeparatorChar.ToString(), new string[upDirectories].Select(_ => "..").ToArray());
+        if (commonLength < pathParts.Length)
+        {
+            if (relativePath.Length > 0)
+            {
+                relativePath += Path.DirectorySeparatorChar;
+            }
+            relativePath += string.Join(Path.DirectorySeparatorChar.ToString(), pathParts.Skip(commonLength));
+        }
+
+        return relativePath;
+    }
 }

src/Mvc/Mvc.Testing/src/PublicAPI.Shipped.txt

@@ -16,6 +16,7 @@ Microsoft.AspNetCore.Mvc.Testing.WebApplicationFactory<TEntryPoint>.CreateDefaul
 Microsoft.AspNetCore.Mvc.Testing.WebApplicationFactory<TEntryPoint>.CreateDefaultClient(System.Uri! baseAddress, params System.Net.Http.DelegatingHandler![]! handlers) -> System.Net.Http.HttpClient!
 Microsoft.AspNetCore.Mvc.Testing.WebApplicationFactory<TEntryPoint>.Dispose() -> void
 Microsoft.AspNetCore.Mvc.Testing.WebApplicationFactory<TEntryPoint>.Factories.get -> System.Collections.Generic.IReadOnlyList<Microsoft.AspNetCore.Mvc.Testing.WebApplicationFactory<TEntryPoint!>!>!
+Microsoft.AspNetCore.Mvc.Testing.WebApplicationFactory<TEntryPoint>.Server.get -> Microsoft.AspNetCore.TestHost.TestServer!
 Microsoft.AspNetCore.Mvc.Testing.WebApplicationFactory<TEntryPoint>.WebApplicationFactory() -> void
 Microsoft.AspNetCore.Mvc.Testing.WebApplicationFactory<TEntryPoint>.WithWebHostBuilder(System.Action<Microsoft.AspNetCore.Hosting.IWebHostBuilder!>! configuration) -> Microsoft.AspNetCore.Mvc.Testing.WebApplicationFactory<TEntryPoint!>!
 Microsoft.AspNetCore.Mvc.Testing.WebApplicationFactoryClientOptions
@@ -40,6 +41,7 @@ virtual Microsoft.AspNetCore.Mvc.Testing.WebApplicationFactory<TEntryPoint>.Conf
 virtual Microsoft.AspNetCore.Mvc.Testing.WebApplicationFactory<TEntryPoint>.ConfigureWebHost(Microsoft.AspNetCore.Hosting.IWebHostBuilder! builder) -> void
 virtual Microsoft.AspNetCore.Mvc.Testing.WebApplicationFactory<TEntryPoint>.CreateHost(Microsoft.Extensions.Hosting.IHostBuilder! builder) -> Microsoft.Extensions.Hosting.IHost!
 virtual Microsoft.AspNetCore.Mvc.Testing.WebApplicationFactory<TEntryPoint>.CreateHostBuilder() -> Microsoft.Extensions.Hosting.IHostBuilder?
+virtual Microsoft.AspNetCore.Mvc.Testing.WebApplicationFactory<TEntryPoint>.CreateServer(Microsoft.AspNetCore.Hosting.IWebHostBuilder! builder) -> Microsoft.AspNetCore.TestHost.TestServer!
 virtual Microsoft.AspNetCore.Mvc.Testing.WebApplicationFactory<TEntryPoint>.CreateWebHostBuilder() -> Microsoft.AspNetCore.Hosting.IWebHostBuilder?
 virtual Microsoft.AspNetCore.Mvc.Testing.WebApplicationFactory<TEntryPoint>.Dispose(bool disposing) -> void
 virtual Microsoft.AspNetCore.Mvc.Testing.WebApplicationFactory<TEntryPoint>.DisposeAsync() -> System.Threading.Tasks.ValueTask

src/Mvc/Mvc.Testing/src/PublicAPI.Unshipped.txt

@@ -1,5 +1,8 @@
 #nullable enable
 Microsoft.AspNetCore.Mvc.Testing.WebApplicationFactory<TEntryPoint>.Initialize() -> void
+*REMOVED*Microsoft.AspNetCore.Mvc.Testing.WebApplicationFactory<TEntryPoint>.Server.get -> Microsoft.AspNetCore.TestHost.TestServer!
 Microsoft.AspNetCore.Mvc.Testing.WebApplicationFactory<TEntryPoint>.Server.get -> Microsoft.AspNetCore.TestHost.TestServer?
 Microsoft.AspNetCore.Mvc.Testing.WebApplicationFactory<TEntryPoint>.TestServer.get -> Microsoft.AspNetCore.TestHost.ITestServer?
+*REMOVED*virtual Microsoft.AspNetCore.Mvc.Testing.WebApplicationFactory<TEntryPoint>.CreateServer(Microsoft.AspNetCore.Hosting.IWebHostBuilder! builder) -> Microsoft.AspNetCore.TestHost.TestServer!
 virtual Microsoft.AspNetCore.Mvc.Testing.WebApplicationFactory<TEntryPoint>.CreateServer(Microsoft.AspNetCore.Hosting.IWebHostBuilder! builder) -> Microsoft.AspNetCore.TestHost.ITestServer!
+

src/Mvc/Mvc.Testing/src/WebApplicationFactory.cs

@@ -246,17 +246,23 @@ private void SetContentRoot(IWebHostBuilder builder)
             return;
         }
 
-        var fromFile = File.Exists("MvcTestingAppManifest.json");
-        var contentRoot = fromFile ? GetContentRootFromFile("MvcTestingAppManifest.json") : GetContentRootFromAssembly();
-
-        if (contentRoot != null)
+        string? contentRoot = null;
+        if (File.Exists("MvcTestingAppManifest.json"))
         {
-            builder.UseContentRoot(contentRoot);
+            var manifestContentRoot = GetContentRootFromFile("MvcTestingAppManifest.json");
+            if (manifestContentRoot is not null && Directory.Exists(manifestContentRoot))
+            {
+                contentRoot = manifestContentRoot;
+            }
         }
-        else
+
+        contentRoot ??= GetContentRootFromAssembly();
+        if (contentRoot is null || !Directory.Exists(contentRoot))
         {
-            builder.UseSolutionRelativeContentRoot(typeof(TEntryPoint).Assembly.GetName().Name!);
+            contentRoot = AppContext.BaseDirectory;
         }
+
+        builder.UseContentRoot(contentRoot);
     }
 
     private static string? GetContentRootFromFile(string file)

src/Mvc/test/Mvc.FunctionalTests/TestingInfrastructureInheritanceTests.cs

@@ -27,8 +27,13 @@ public void TestingInfrastructure_WebHost_WithWebHostBuilderRespectsCustomizatio
         Assert.Equal(new[] { "ConfigureWebHost", "Customization", "FurtherCustomization" }, factory.ConfigureWebHostCalled.ToArray());
         Assert.True(factory.CreateServerCalled);
         Assert.True(factory.CreateWebHostBuilderCalled);
-        // GetTestAssemblies is not called when reading content roots from MvcAppManifest
-        Assert.False(factory.GetTestAssembliesCalled);
+
+        // When run locally on the developer machine, the test manifest will be generated,
+        // and the content root will be read from the manifest file.
+        // However, when run in CI, the relative path in the metadata will not exist,
+        // and the content root lookup logic will probe the test assemblies too.
+        //Assert.False(factory.GetTestAssembliesCalled);
+
         Assert.True(factory.CreateHostBuilderCalled);
         Assert.False(factory.CreateHostCalled);
     }
@@ -45,7 +50,13 @@ public void TestingInfrastructure_GenericHost_WithWithHostBuilderRespectsCustomi
 
         // Assert
         Assert.Equal(new[] { "ConfigureWebHost", "Customization", "FurtherCustomization" }, factory.ConfigureWebHostCalled.ToArray());
-        Assert.False(factory.GetTestAssembliesCalled);
+
+        // When run locally on the developer machine, the test manifest will be generated,
+        // and the content root will be read from the manifest file.
+        // However, when run in CI, the relative path in the metadata will not exist,
+        // and the content root lookup logic will probe the test assemblies too.
+        //Assert.False(factory.GetTestAssembliesCalled);
+
         Assert.True(factory.CreateHostBuilderCalled);
         Assert.True(factory.CreateHostCalled);
         Assert.False(factory.CreateServerCalled);
@@ -128,7 +139,12 @@ public void Dispose()
 
     private class CustomizedFactory<TEntryPoint> : WebApplicationFactory<TEntryPoint> where TEntryPoint : class
     {
+        // GetTestAssemblies is not called when reading content roots from MvcAppManifest,
+        // and the content root specified in the manifest is a path that exists.
+        // Otherwise, the WebApplicationFactory will try to look for the referenced assemblies which have
+        // `WebApplicationFactoryContentRootAttribute` applied to them, to extract the content root path from that metadata.
         public bool GetTestAssembliesCalled { get; private set; }
+
         public bool CreateWebHostBuilderCalled { get; private set; }
         public bool CreateHostBuilderCalled { get; private set; }
         public bool CreateServerCalled { get; private set; }