Skip to content

Commit 1e7ee6f

Browse files
agockeagocke
authored
Move hosting tests to XUnit 3 (#120234)
The primary advantage in this change is that test assemblies are now executables, not libraries. This makes debugging easier, as the executable can be run directly under the debugger. It also makes deployment of the tests easier, as we only need to copy the executable and set DOTNET_ROOT to the pre-built framework. This is also a kind of trial-balloon. Many other tests (esp. libraries tests) in this repo would benefit from the execution advantages. However, they are built in more configurations today and are more complex. The host tests are an easier lift. There are a few breaking changes that have been fixed by this PR: * xunit3 already defines a class called TestContext, so I needed to rename the existing TestContext to HostTestContext to avoid conflicts * Some command-line switches have changed names * xunit3 now supports an Assert.Skip for runtime skipping. I've utilized this in a few places (especially class constructors) to replace PlatformSpecific on classes, or simplify the skip handling.
1 parent 9b8304c commit 1e7ee6f

File tree

62 files changed

+381
-341
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

62 files changed

+381
-341
lines changed

‎src/installer/tests/AppHost.Bundle.Tests/AppHost.Bundle.Tests.csproj‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,17 @@
22

33
<PropertyGroup>
44
<Description>Apphost Bundle Tests</Description>
5+
<OutputType>Exe</OutputType>
6+
<RuntimeIdentifier>$(TargetRid)</RuntimeIdentifier>
57
<TargetFramework>$(TestInfraTargetFramework)</TargetFramework>
68
<AssemblyName>AppHost.Bundle.Tests</AssemblyName>
79
<GenerateRuntimeConfigurationFiles>true</GenerateRuntimeConfigurationFiles>
810
<!-- Reduce the length of the test output dir to make it more reliable on Windows. -->
911
<TestsOutputName>ahb</TestsOutputName>
1012
<!-- NuGet warns about a transitive P2P to System.Text.Json that can't be pruned.
1113
This is a false positive: https://github.com/NuGet/Home/issues/14103 -->
12-
<NoWarn>$(NoWarn);NU1511</NoWarn>
14+
<NoWarn>$(NoWarn);NU1511;xUnit1004</NoWarn>
15+
<TestRunnerName>XUnitV3</TestRunnerName>
1316
</PropertyGroup>
1417

1518
<ItemGroup>

‎src/installer/tests/AppHost.Bundle.Tests/AppLaunch.cs‎

Lines changed: 17 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ private void RunTheApp(string path, bool selfContained)
2727
Command.Create(path)
2828
.CaptureStdErr()
2929
.CaptureStdOut()
30-
.DotNetRoot(selfContained ? null : TestContext.BuiltDotNet.BinPath)
30+
.DotNetRoot(selfContained ? null : HostTestContext.BuiltDotNet.BinPath)
3131
.Execute()
3232
.Should().Pass()
3333
.And.HaveStdOutContaining("Hello World!");
@@ -75,9 +75,9 @@ private void RunApp(bool selfContained)
7575
if (OperatingSystem.IsWindows())
7676
{
7777
// StandaloneApp sets FileVersion to NETCoreApp version. On Windows, this should be copied to singlefilehost resources.
78-
string expectedVersion = TestContext.MicrosoftNETCoreAppVersion.Contains('-')
79-
? TestContext.MicrosoftNETCoreAppVersion[..TestContext.MicrosoftNETCoreAppVersion.IndexOf('-')]
80-
: TestContext.MicrosoftNETCoreAppVersion;
78+
string expectedVersion = HostTestContext.MicrosoftNETCoreAppVersion.Contains('-')
79+
? HostTestContext.MicrosoftNETCoreAppVersion[..HostTestContext.MicrosoftNETCoreAppVersion.IndexOf('-')]
80+
: HostTestContext.MicrosoftNETCoreAppVersion;
8181
Assert.Equal(expectedVersion, System.Diagnostics.FileVersionInfo.GetVersionInfo(singleFile).FileVersion);
8282
}
8383
}
@@ -103,14 +103,17 @@ private void NonAsciiCharacterSelfContainedApp()
103103
if (OperatingSystem.IsWindows())
104104
{
105105
// StandaloneApp sets FileVersion to NETCoreApp version. On Windows, this should be copied to singlefilehost resources.
106-
string expectedVersion = TestContext.MicrosoftNETCoreAppVersion.Contains('-')
107-
? TestContext.MicrosoftNETCoreAppVersion[..TestContext.MicrosoftNETCoreAppVersion.IndexOf('-')]
108-
: TestContext.MicrosoftNETCoreAppVersion;
106+
string expectedVersion = HostTestContext.MicrosoftNETCoreAppVersion.Contains('-')
107+
? HostTestContext.MicrosoftNETCoreAppVersion[..HostTestContext.MicrosoftNETCoreAppVersion.IndexOf('-')]
108+
: HostTestContext.MicrosoftNETCoreAppVersion;
109109
Assert.Equal(expectedVersion, System.Diagnostics.FileVersionInfo.GetVersionInfo(singleFile).FileVersion);
110110
}
111111
}
112112

113-
[ConditionalTheory(typeof(Binaries.CetCompat), nameof(Binaries.CetCompat.IsSupported))]
113+
[Theory(
114+
SkipType = typeof(Binaries.CetCompat),
115+
SkipUnless = nameof(Binaries.CetCompat.IsSupported),
116+
Skip = "CET is not supported on this platform")]
114117
[InlineData(true)]
115118
[InlineData(false)]
116119
public void DisableCetCompat(bool selfContained)
@@ -124,12 +127,12 @@ public void DisableCetCompat(bool selfContained)
124127
Command.Create(singleFile)
125128
.CaptureStdErr()
126129
.CaptureStdOut()
127-
.DotNetRoot(TestContext.BuiltDotNet.BinPath, TestContext.BuildArchitecture)
130+
.DotNetRoot(HostTestContext.BuiltDotNet.BinPath, HostTestContext.BuildArchitecture)
128131
.MultilevelLookup(false)
129132
.Execute()
130133
.Should().Pass()
131134
.And.HaveStdOutContaining("Hello World")
132-
.And.HaveStdOutContaining(TestContext.MicrosoftNETCoreAppVersion);
135+
.And.HaveStdOutContaining(HostTestContext.MicrosoftNETCoreAppVersion);
133136
}
134137

135138
[Theory]
@@ -158,7 +161,7 @@ public void FrameworkDependent_NoBundleEntryPoint()
158161

159162
using (var dotnetWithMockHostFxr = TestArtifact.Create("mockhostfxrFrameworkMissingFailure"))
160163
{
161-
var dotnet = new DotNetBuilder(dotnetWithMockHostFxr.Location, TestContext.BuiltDotNet.BinPath, null)
164+
var dotnet = new DotNetBuilder(dotnetWithMockHostFxr.Location, HostTestContext.BuiltDotNet.BinPath, null)
162165
.RemoveHostFxr()
163166
.AddMockHostFxr(new Version(2, 2, 0))
164167
.Build();
@@ -177,7 +180,7 @@ public void FrameworkDependent_NoBundleEntryPoint()
177180
}
178181

179182
[Fact]
180-
[PlatformSpecific(TestPlatforms.Windows)] // GUI app host is only supported on Windows.
183+
[PlatformSpecific(TestPlatforms.Windows)]
181184
public void FrameworkDependent_GUI_DownlevelHostFxr_ErrorDialog()
182185
{
183186
var singleFile = sharedTestState.FrameworkDependentApp.Bundle();
@@ -188,14 +191,14 @@ public void FrameworkDependent_GUI_DownlevelHostFxr_ErrorDialog()
188191
{
189192
string expectedErrorCode = Constants.ErrorCode.BundleExtractionFailure.ToString("x");
190193

191-
var dotnet = new DotNetBuilder(dotnetWithMockHostFxr.Location, TestContext.BuiltDotNet.BinPath, null)
194+
var dotnet = new DotNetBuilder(dotnetWithMockHostFxr.Location, HostTestContext.BuiltDotNet.BinPath, null)
192195
.RemoveHostFxr()
193196
.AddMockHostFxr(new Version(5, 0, 0))
194197
.Build();
195198

196199
Command command = Command.Create(singleFile)
197200
.EnableTracingAndCaptureOutputs()
198-
.DotNetRoot(dotnet.BinPath, TestContext.BuildArchitecture)
201+
.DotNetRoot(dotnet.BinPath, HostTestContext.BuildArchitecture)
199202
.Start();
200203

201204
WindowsUtils.WaitForPopupFromProcess(command.Process);

‎src/installer/tests/AppHost.Bundle.Tests/BundledAppWithSubDirs.cs‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ private FluentAssertions.AndConstraint<CommandResultAssertions> RunTheApp(string
2424
{
2525
CommandResult result = Command.Create(path)
2626
.EnableTracingAndCaptureOutputs()
27-
.DotNetRoot(selfContained ? null : TestContext.BuiltDotNet.BinPath)
27+
.DotNetRoot(selfContained ? null : HostTestContext.BuiltDotNet.BinPath)
2828
.MultilevelLookup(false)
2929
.Execute();
3030
if (deleteApp)
@@ -142,8 +142,7 @@ public void FrameworkDependent_Targeting50(BundleOptions options)
142142
}
143143
}
144144

145-
[Fact]
146-
[ActiveIssue("https://github.com/dotnet/runtime/issues/54234")]
145+
[Fact(Skip = "https://github.com/dotnet/runtime/issues/54234")]
147146
// NOTE: when enabling this test take a look at commented code marked by "ACTIVE ISSUE:" in SharedTestState
148147
public void SelfContained_R2R_Composite()
149148
{

‎src/installer/tests/AppHost.Bundle.Tests/NativeLibraries.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ private void PInvoke(bool selfContained, bool bundleNative)
3838
Command.Create(app, "load_native_library_pinvoke")
3939
.CaptureStdErr()
4040
.CaptureStdOut()
41-
.DotNetRoot(selfContained ? null : TestContext.BuiltDotNet.BinPath)
41+
.DotNetRoot(selfContained ? null : HostTestContext.BuiltDotNet.BinPath)
4242
// Specify an extraction root that will get cleaned up by the test app artifact
4343
.EnvironmentVariable(Constants.BundleExtractBase.EnvironmentVariable, extractionRoot)
4444
.Execute()
@@ -66,7 +66,7 @@ private void TryLoad(bool selfContained, bool bundleNative)
6666
Command.Create(app, "load_native_library_api")
6767
.CaptureStdErr()
6868
.CaptureStdOut()
69-
.DotNetRoot(selfContained ? null : TestContext.BuiltDotNet.BinPath)
69+
.DotNetRoot(selfContained ? null : HostTestContext.BuiltDotNet.BinPath)
7070
// Specify an extraction root that will get cleaned up by the test app artifact
7171
.EnvironmentVariable(Constants.BundleExtractBase.EnvironmentVariable, extractionRoot)
7272
.Execute()

‎src/installer/tests/Directory.Build.props‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,11 @@
44
<PropertyGroup>
55
<TestArtifactsOutputRoot>$(ArtifactsDir)tests\host\$(TargetOS).$(TargetArchitecture).$(Configuration)\</TestArtifactsOutputRoot>
66
<TestInfraTargetFramework>$(NetCoreAppToolCurrent)</TestInfraTargetFramework>
7-
<TestCaseFilter>category!=failing</TestCaseFilter>
8-
<TestRunnerAdditionalArguments>--filter $(TestCaseFilter) -v detailed</TestRunnerAdditionalArguments>
7+
<TestCaseFilter>/[category!=failing]</TestCaseFilter>
8+
<TestRunnerAdditionalArguments>--filter-query $(TestCaseFilter)</TestRunnerAdditionalArguments>
99
<!-- Enable crash and hang dumps -->
10-
<TestRunnerAdditionalArguments>$(TestRunnerAdditionalArguments) --blame-crash-dump-type full</TestRunnerAdditionalArguments>
11-
<TestRunnerAdditionalArguments>$(TestRunnerAdditionalArguments) --blame-hang-timeout 5m --blame-hang-dump-type full</TestRunnerAdditionalArguments>
12-
<UseVSTestRunner>true</UseVSTestRunner>
10+
<TestRunnerAdditionalArguments>$(TestRunnerAdditionalArguments) --crashdump</TestRunnerAdditionalArguments>
11+
<TestRunnerAdditionalArguments>$(TestRunnerAdditionalArguments) --hangdump --hangdump-timeout 5m --hangdump-type full</TestRunnerAdditionalArguments>
1312
<RunAnalyzers>false</RunAnalyzers>
1413
</PropertyGroup>
1514

‎src/installer/tests/Directory.Build.targets‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,11 @@
9292
<RunSettingsTestCaseFilter>$(TestCaseFilter)</RunSettingsTestCaseFilter>
9393
</PropertyGroup>
9494

95+
<ItemGroup Condition="'$(IsTestProject)' == 'true'">
96+
<PackageReference Include="Microsoft.Testing.Extensions.CrashDump" Version="1.7.3" />
97+
<PackageReference Include="Microsoft.Testing.Extensions.HangDump" Version="1.7.3" />
98+
</ItemGroup>
99+
95100
<Import Project="$(RepositoryEngineeringDir)testing\runsettings.targets"
96101
Condition="'$(IsTestProject)' == 'true'" />
97102

‎src/installer/tests/HostActivation.Tests/Breadcrumbs.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ public Breadcrumbs(SharedTestState fixture)
2222
[Fact]
2323
public void BreadcrumbThreadFinishes()
2424
{
25-
TestContext.BuiltDotNet.Exec(sharedTestState.App.AppDll)
25+
HostTestContext.BuiltDotNet.Exec(sharedTestState.App.AppDll)
2626
.EnvironmentVariable(Constants.Breadcrumbs.EnvironmentVariable, sharedTestState.BreadcrumbLocation)
2727
.EnableTracingAndCaptureOutputs()
2828
.Execute()
@@ -34,7 +34,7 @@ public void BreadcrumbThreadFinishes()
3434
[Fact]
3535
public void UnhandledException_BreadcrumbThreadDoesNotFinish()
3636
{
37-
TestContext.BuiltDotNet.Exec(sharedTestState.App.AppDll, "throw_exception")
37+
HostTestContext.BuiltDotNet.Exec(sharedTestState.App.AppDll, "throw_exception")
3838
.EnvironmentVariable(Constants.Breadcrumbs.EnvironmentVariable, sharedTestState.BreadcrumbLocation)
3939
.EnableTracingAndCaptureOutputs()
4040
.DisableDumps() // Expected to throw an exception

‎src/installer/tests/HostActivation.Tests/DependencyResolution/AdditionalProbingPath.cs‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -111,24 +111,24 @@ public class SharedTestState : SharedTestStateBase
111111
public SharedTestState()
112112
{
113113
DotNetWithNetCoreApp = DotNet("WithNetCoreApp")
114-
.AddMicrosoftNETCoreAppFrameworkMockCoreClr(TestContext.MicrosoftNETCoreAppVersion)
114+
.AddMicrosoftNETCoreAppFrameworkMockCoreClr(HostTestContext.MicrosoftNETCoreAppVersion)
115115
.Build();
116116

117-
string nativeDependencyRelPath = $"{TestContext.BuildRID}/{Binaries.GetSharedLibraryFileNameForCurrentPlatform("native")}";
118-
FrameworkReferenceApp = CreateFrameworkReferenceApp(Constants.MicrosoftNETCoreApp, TestContext.MicrosoftNETCoreAppVersion, b => b
117+
string nativeDependencyRelPath = $"{HostTestContext.BuildRID}/{Binaries.GetSharedLibraryFileNameForCurrentPlatform("native")}";
118+
FrameworkReferenceApp = CreateFrameworkReferenceApp(Constants.MicrosoftNETCoreApp, HostTestContext.MicrosoftNETCoreAppVersion, b => b
119119
.WithProject(DependencyName, DependencyVersion, p => p
120120
.WithAssemblyGroup(null, g => g
121121
.WithAsset($"{DependencyName}.dll", f => f.NotOnDisk()))
122-
.WithNativeLibraryGroup(TestContext.BuildRID, g => g
122+
.WithNativeLibraryGroup(HostTestContext.BuildRID, g => g
123123
.WithAsset(nativeDependencyRelPath, f => f.NotOnDisk()))));
124124
RuntimeConfig.FromFile(FrameworkReferenceApp.RuntimeConfigJson)
125-
.WithTfm(TestContext.Tfm)
125+
.WithTfm(HostTestContext.Tfm)
126126
.Save();
127127

128128
AdditionalProbingPath = Path.Combine(Location, "probe");
129129
(DependencyPath, NativeDependencyDirectory) = AddDependencies(AdditionalProbingPath);
130130

131-
AdditionalProbingPath_ArchTfm = Path.Combine(AdditionalProbingPath, TestContext.BuildArchitecture, TestContext.Tfm);
131+
AdditionalProbingPath_ArchTfm = Path.Combine(AdditionalProbingPath, HostTestContext.BuildArchitecture, HostTestContext.Tfm);
132132
(DependencyPath_ArchTfm, NativeDependencyDirectory_ArchTfm) = AddDependencies(AdditionalProbingPath_ArchTfm);
133133

134134
(string, string) AddDependencies(string probeDir)

‎src/installer/tests/HostActivation.Tests/DependencyResolution/DepsFile.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -92,10 +92,10 @@ public class SharedTestState : SharedTestStateBase
9292
public SharedTestState()
9393
{
9494
DotNetWithNetCoreApp = DotNet("WithNetCoreApp")
95-
.AddMicrosoftNETCoreAppFrameworkMockCoreClr(TestContext.MicrosoftNETCoreAppVersion)
95+
.AddMicrosoftNETCoreAppFrameworkMockCoreClr(HostTestContext.MicrosoftNETCoreAppVersion)
9696
.Build();
9797

98-
FrameworkReferenceApp = CreateFrameworkReferenceApp(Constants.MicrosoftNETCoreApp, TestContext.MicrosoftNETCoreAppVersion, b => b
98+
FrameworkReferenceApp = CreateFrameworkReferenceApp(Constants.MicrosoftNETCoreApp, HostTestContext.MicrosoftNETCoreAppVersion, b => b
9999
.WithProject(DependencyName, "1.0.0", p => p
100100
.WithAssemblyGroup(null, g => g.WithAsset($"{DependencyName}.dll"))));
101101

‎src/installer/tests/HostActivation.Tests/DependencyResolution/LocalPath.cs‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -66,8 +66,8 @@ private void RuntimeAssemblies(bool isSelfContained, bool useLocalPath)
6666
{
6767
// Add RID-specific assembly
6868
(string ridPath, string localRidPath) = GetPaths(libraryType, true);
69-
b.WithRuntimeLibrary(libraryType, $"{library}-{TestContext.BuildRID}", "1.0.0", p => p
70-
.WithAssemblyGroup(TestContext.BuildRID, g => g
69+
b.WithRuntimeLibrary(libraryType, $"{library}-{HostTestContext.BuildRID}", "1.0.0", p => p
70+
.WithAssemblyGroup(HostTestContext.BuildRID, g => g
7171
.WithAsset(ridPath, useLocalPath ? f => f.WithLocalPath(localRidPath) : null)));
7272
}
7373
}
@@ -114,7 +114,7 @@ private void RuntimeAssemblies(bool isSelfContained, bool useLocalPath)
114114
static (string Path, string LocalPath) GetPaths(RuntimeLibraryType libraryType, bool useRid)
115115
{
116116
string library = $"Test{libraryType}";
117-
string path = useRid ? $"lib/{TestContext.BuildRID}/{library}-{TestContext.BuildRID}.dll" : $"lib/{library}.dll";
117+
string path = useRid ? $"lib/{HostTestContext.BuildRID}/{library}-{HostTestContext.BuildRID}.dll" : $"lib/{library}.dll";
118118
return (path, $"{libraryType}/{path}");
119119
}
120120
}
@@ -137,9 +137,9 @@ private void NativeLibraries(bool isSelfContained, bool useLocalPath)
137137
{
138138
// Add RID-specific native library
139139
(string ridPath, string localRidPath) = GetPaths(libraryType, true);
140-
b.WithRuntimeLibrary(libraryType, $"{library}-{TestContext.BuildRID}", "1.0.0", p => p
141-
.WithNativeLibraryGroup(TestContext.BuildRID, g => g
142-
.WithAsset($"{ridPath}/{library}-{TestContext.BuildRID}.native", useLocalPath ? f => f.WithLocalPath($"{localRidPath}/{library}-{TestContext.BuildRID}.native") : null)));
140+
b.WithRuntimeLibrary(libraryType, $"{library}-{HostTestContext.BuildRID}", "1.0.0", p => p
141+
.WithNativeLibraryGroup(HostTestContext.BuildRID, g => g
142+
.WithAsset($"{ridPath}/{library}-{HostTestContext.BuildRID}.native", useLocalPath ? f => f.WithLocalPath($"{localRidPath}/{library}-{HostTestContext.BuildRID}.native") : null)));
143143
}
144144
}
145145

@@ -184,7 +184,7 @@ private void NativeLibraries(bool isSelfContained, bool useLocalPath)
184184

185185
static (string Path, string LocalPath) GetPaths(NetCoreAppBuilder.RuntimeLibraryType libraryType, bool useRid)
186186
{
187-
string path = useRid ? $"native/{TestContext.BuildRID}" : "native";
187+
string path = useRid ? $"native/{HostTestContext.BuildRID}" : "native";
188188
return (path, $"{libraryType}/{path}");
189189
}
190190
}
@@ -245,7 +245,7 @@ private static TestApp CreateApp(bool isSelfContained, Action<NetCoreAppBuilder>
245245
}
246246
else
247247
{
248-
app.PopulateFrameworkDependent(Constants.MicrosoftNETCoreApp, TestContext.MicrosoftNETCoreAppVersion, customizer);
248+
app.PopulateFrameworkDependent(Constants.MicrosoftNETCoreApp, HostTestContext.MicrosoftNETCoreAppVersion, customizer);
249249
}
250250
return app;
251251
}
@@ -257,7 +257,7 @@ public class SharedTestState : SharedTestStateBase
257257
public SharedTestState()
258258
{
259259
DotNetWithNetCoreApp = DotNet("WithNetCoreApp")
260-
.AddMicrosoftNETCoreAppFrameworkMockCoreClr(TestContext.MicrosoftNETCoreAppVersion)
260+
.AddMicrosoftNETCoreAppFrameworkMockCoreClr(HostTestContext.MicrosoftNETCoreAppVersion)
261261
.Build();
262262
}
263263
}

0 commit comments

Comments
 (0)