Skip to content

Commit 1081a93

Browse files
committed
Improve test reliability, diagnostics, and cleanup
- Enhance test output with detailed diagnostics and error context - Use robust assertions with informative failure messages - Create temp test projects under artifacts/tmp for isolation - Add retries and file attribute handling to test dir cleanup - Check for test executable and coverlet.MTP.dll before running - Refactor HelpCommandTests for consistent path handling - Fix sample class naming mismatch in test project - Add condition to MSBuild import for props file robustness - Update NuGet config and project file handling for clarity
1 parent b68b79a commit 1081a93

File tree

14 files changed

+121
-81
lines changed

14 files changed

+121
-81
lines changed

eng/build.yml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,8 @@ steps:
3939
displayName: Pack
4040

4141
- script: |
42+
artifacts\bin\coverlet.MTP.unit.tests\debug\coverlet.MTP.unit.tests.exe --diagnostic --diagnostic-verbosity $(BuildConfiguration) --report-xunit-trx --report-xunit-trx-filename "coverlet.MTP.unit.tests.trx" --diagnostic --diagnostic-output-directory "$(Build.SourcesDirectory)/artifacts/log/$(BuildConfiguration)"
43+
artifacts\bin\coverlet.MTP.validation.tests\debug\coverlet.MTP.validation.tests.exe --diagnostic --diagnostic-verbosity $(BuildConfiguration) --report-xunit-trx --report-xunit-trx-filename "coverlet.MTP.validation.tests.trx" --diagnostic --diagnostic-output-directory "$(Build.SourcesDirectory)/artifacts/log/$(BuildConfiguration)"
4244
dotnet test test/coverlet.core.tests/coverlet.core.tests.csproj -c $(BuildConfiguration) --no-build -bl:test.core.binlog /p:CollectCoverage=true /p:CoverletOutputFormat=opencover /p:Exclude="[coverlet.core.tests.samples.netstandard]*%2c[coverlet.tests.projectsample]*" /p:ExcludeByAttribute="GeneratedCodeAttribute" --diag:"$(Build.SourcesDirectory)/artifacts/log/coverlet.core.tests.diag.$(BuildConfiguration).log;tracelevel=verbose"
4345
dotnet test test/coverlet.core.coverage.tests/coverlet.core.coverage.tests.csproj -c $(BuildConfiguration) --no-build -bl:test.core.coverage.binlog /p:CollectCoverage=true /p:CoverletOutputFormat=opencover /p:Exclude="[coverlet.core.tests.samples.netstandard]*%2c[coverlet.tests.projectsample]*" /p:ExcludeByAttribute="GeneratedCodeAttribute" -- --results-directory "$(Build.SourcesDirectory))/artifacts/reports" --report-xunit-trx --report-xunit-trx-filename "coverlet.core.coverage.tests.trx" --diagnostic-verbosity debug --diagnostic --diagnostic-output-directory "$(Build.SourcesDirectory)/artifacts/log/$(BuildConfiguration)"
4446
dotnet test test/coverlet.msbuild.tasks.tests\coverlet.msbuild.tasks.tests.csproj -c $(BuildConfiguration) --no-build -bl:test.msbuild.tasks.binlog /p:CollectCoverage=true /p:CoverletOutputFormat=opencover /p:Exclude="[coverlet.core.tests.samples.netstandard]*%2c[coverlet.tests.projectsample]*" /p:ExcludeByAttribute="GeneratedCodeAttribute" --diag:"$(Build.SourcesDirectory)/artifacts/log/coverlet.msbuild.tasks.tests.diag.$(BuildConfiguration).log;tracelevel=verbose"
Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
<Project>
2-
<Import Project="$(MSBuildThisFileDirectory)..\..\buildMultiTargeting\coverlet.props" />
2+
<Import Project="$(MSBuildThisFileDirectory)..\..\buildMultiTargeting\coverlet.MTP.props" />
33
</Project>
Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
<Project>
2-
<Import Project="$(MSBuildThisFileDirectory)..\..\buildMultiTargeting\coverlet.targets" />
2+
<Import Project="$(MSBuildThisFileDirectory)..\..\buildMultiTargeting\coverlet.MTP.targets" />
33
</Project>
Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
11
<Project>
2-
<Import Project="$(MSBuildThisFileDirectory)..\..\buildMultiTargeting\coverlet.props" />
2+
<Import Project="$(MSBuildThisFileDirectory)..\..\buildMultiTargeting\coverlet.MTP.props"
3+
Condition="Exists('$(MSBuildThisFileDirectory)..\..\buildMultiTargeting\coverlet.MTP.props')" />
34
</Project>
Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
<Project>
2-
<Import Project="$(MSBuildThisFileDirectory)..\..\buildMultiTargeting\coverlet.targets" />
2+
<Import Project="$(MSBuildThisFileDirectory)..\..\buildMultiTargeting\coverlet.MTP.targets" />
33
</Project>

src/coverlet.MTP/coverlet.MTP.csproj

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,14 +12,22 @@
1212

1313
<SuppressDependenciesWhenPacking>false</SuppressDependenciesWhenPacking>
1414
<TargetsForTfmSpecificContentInPackage>$(TargetsForTfmSpecificContentInPackage);CopyProjectReferencesToPackage</TargetsForTfmSpecificContentInPackage>
15-
15+
1616
<EnablePackageValidation>false</EnablePackageValidation>
1717
<!--<IsTestProject>true</IsTestProject>-->
1818
<TestingPlatformDotnetTestSupport>true</TestingPlatformDotnetTestSupport>
1919
<IsPackable>true</IsPackable>
2020
<NoPackageAnalysis>true</NoPackageAnalysis>
2121
<!-- disable transitive version update and use versions defined in coverlet.core -->
2222
<CentralPackageTransitivePinningEnabled>false</CentralPackageTransitivePinningEnabled>
23+
<!-- settings for Muse.coverlet.MTP.globalconfig -->
24+
<!-- <CodeAnalysisTreatWarningsAsErrors>false</CodeAnalysisTreatWarningsAsErrors>
25+
<AnalysisMode>Recommended</AnalysisMode>
26+
<AnalysisLevel>latest-major</AnalysisLevel>
27+
<Deterministic>true</Deterministic>
28+
<EnableNETAnalyzers>true</EnableNETAnalyzers>
29+
<EnforceCodeStyleInBuild>true</EnforceCodeStyleInBuild>
30+
<EmbedUntrackedSources>true</EmbedUntrackedSources> -->
2331
</PropertyGroup>
2432

2533
<PropertyGroup>
@@ -56,6 +64,10 @@
5664
<ProjectReference Include="$(MSBuildThisFileDirectory)..\coverlet.core\coverlet.core.csproj" PrivateAssets="all" />
5765
</ItemGroup>
5866

67+
<!-- <ItemGroup>
68+
<GlobalAnalyzerConfigFiles Include="$(MSBuildThisFileDirectory)../contentFiles/any/any/Muse.coverlet.MTP.globalconfig" />
69+
</ItemGroup> -->
70+
5971
<!-- NuGet package layout -->
6072
<!-- NuGet folders https://learn.microsoft.com/nuget/create-packages/creating-a-package#from-a-convention-based-working-directory -->
6173
<ItemGroup>
@@ -91,6 +103,6 @@
91103
<!-- Print batches for debug purposes -->
92104
<Message Text="Batch for .nupkg: ReferenceCopyLocalPaths = @(_ReferenceCopyLocalPaths), ReferenceCopyLocalPaths.DestinationSubDirectory = %(_ReferenceCopyLocalPaths.DestinationSubDirectory) Filename = %(_ReferenceCopyLocalPaths.Filename) Extension = %(_ReferenceCopyLocalPaths.Extension)" Importance="High" Condition="'@(_ReferenceCopyLocalPaths)' != ''" />
93105

94-
</Target>
106+
</Target>
95107

96108
</Project>

test/coverlet.MTP.validation.tests/CollectCoverageTests.cs

Lines changed: 82 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -20,15 +20,16 @@ public class CollectCoverageTests
2020
private readonly string _localPackagesPath;
2121
private const string CoverageJsonFileName = "coverage.json";
2222
private const string CoverageCoberturaFileName = "coverage.cobertura.xml";
23+
private readonly string _repoRoot;
2324

2425
public CollectCoverageTests()
2526
{
2627
_buildConfiguration = "Debug";
2728
_buildTargetFramework = "net8.0";
2829

2930
// Get local packages path (adjust based on your build output)
30-
string repoRoot = Path.GetFullPath(Path.Combine(Directory.GetCurrentDirectory(), "..", "..", "..", ".."));
31-
_localPackagesPath = Path.Combine(repoRoot, "artifacts", "package", _buildConfiguration.ToLowerInvariant());
31+
_repoRoot = Path.GetFullPath(Path.Combine(Directory.GetCurrentDirectory(), "..", "..", "..", ".."));
32+
_localPackagesPath = Path.Combine(_repoRoot, "artifacts", "package", _buildConfiguration.ToLowerInvariant());
3233
}
3334

3435
[Fact]
@@ -41,12 +42,10 @@ public async Task BasicCoverage_CollectsDataForCoveredLines()
4142
// Act
4243
var result = await RunTestsWithCoverage(testProject.ProjectPath, "--coverage");
4344

44-
TestContext.Current.AddAttachment(
45-
"Test Output",
46-
result.CombinedOutput);
45+
TestContext.Current?.AddAttachment("Test Output", result.CombinedOutput);
4746

4847
// Assert
49-
Assert.Equal(0, result.ExitCode);
48+
Assert.True( result.ExitCode == 0, $"Expected successful test run (exit code 0) but got {result.ExitCode}.\n\n{result.CombinedOutput}");
5049
Assert.Contains("Passed!", result.StandardOutput);
5150

5251
string[] coverageFiles = Directory.GetFiles(testProject.OutputDirectory, CoverageJsonFileName, SearchOption.AllDirectories);
@@ -69,12 +68,10 @@ public async Task CoverageWithFormat_GeneratesCorrectOutputFormat()
6968
testProject.ProjectPath,
7069
"--coverage --coverage-output-format cobertura");
7170

72-
TestContext.Current.AddAttachment(
73-
"Test Output",
74-
result.CombinedOutput);
71+
TestContext.Current?.AddAttachment("Test Output", result.CombinedOutput);
7572

7673
// Assert
77-
Assert.Equal(0, result.ExitCode);
74+
Assert.True(result.ExitCode == 0, $"Expected successful test run (exit code 0) but got {result.ExitCode}.\n\n{result.CombinedOutput}");
7875

7976
string[] coverageFiles = Directory.GetFiles(testProject.OutputDirectory, CoverageCoberturaFileName, SearchOption.AllDirectories);
8077
Assert.NotEmpty(coverageFiles);
@@ -94,12 +91,10 @@ public async Task CoverageInstrumentation_TracksMethodHits()
9491
// Act
9592
var result = await RunTestsWithCoverage(testProject.ProjectPath, "--coverage");
9693

97-
TestContext.Current.AddAttachment(
98-
"Test Output",
99-
result.CombinedOutput);
94+
TestContext.Current?.AddAttachment("Test Output", result.CombinedOutput);
10095

10196
// Assert
102-
Assert.Equal(0, result.ExitCode);
97+
Assert.True(result.ExitCode == 0, $"Expected successful test run (exit code 0) but got {result.ExitCode}.\n\n{result.CombinedOutput}");
10398

10499
string[] coverageFiles = Directory.GetFiles(testProject.OutputDirectory, CoverageJsonFileName, SearchOption.AllDirectories);
105100
var coverageData = ParseCoverageJson(coverageFiles[0]);
@@ -149,12 +144,10 @@ public async Task BranchCoverage_TracksConditionalPaths()
149144
// Act
150145
var result = await RunTestsWithCoverage(testProject.ProjectPath, "--coverage");
151146

152-
TestContext.Current.AddAttachment(
153-
"Test Output",
154-
result.CombinedOutput);
147+
TestContext.Current?.AddAttachment("Test Output", result.CombinedOutput);
155148

156149
// Assert
157-
Assert.Equal(0, result.ExitCode);
150+
Assert.True(result.ExitCode == 0, $"Expected successful test run (exit code 0) but got {result.ExitCode}.\n\n{result.CombinedOutput}");
158151

159152
string[] coverageFiles = Directory.GetFiles(testProject.OutputDirectory, CoverageJsonFileName, SearchOption.AllDirectories);
160153
var coverageData = ParseCoverageJson(coverageFiles[0]);
@@ -207,12 +200,10 @@ public async Task MultipleCoverageFormats_GeneratesAllReports()
207200
testProject.ProjectPath,
208201
"--coverage --coverage-output-format json,cobertura,lcov");
209202

210-
TestContext.Current.AddAttachment(
211-
"Test Output",
212-
result.CombinedOutput);
203+
TestContext.Current?.AddAttachment("Test Output", result.CombinedOutput);
213204

214205
// Assert
215-
Assert.Equal(0, result.ExitCode);
206+
Assert.True(result.ExitCode == 0, $"Expected successful test run (exit code 0) but got {result.ExitCode}.\n\n{result.CombinedOutput}");
216207

217208
// Verify all formats are generated
218209
Assert.NotEmpty(Directory.GetFiles(testProject.OutputDirectory, "coverage.json", SearchOption.AllDirectories));
@@ -223,14 +214,19 @@ public async Task MultipleCoverageFormats_GeneratesAllReports()
223214
#region Helper Methods
224215

225216
private TestProject CreateTestProject(
217+
226218
bool includeSimpleTest = false,
227219
bool includeMethodTests = false,
228220
bool includeMultipleClasses = false,
229221
bool includeCalculatorTest = false,
230222
bool includeBranchTest = false,
231223
bool includeMultipleTests = false)
232224
{
233-
string tempPath = Path.Combine(Path.GetTempPath(), $"CoverletMTP_Test_{Guid.NewGuid():N}");
225+
// Use repository artifacts folder instead of user temp
226+
string artifactsTemp = Path.Combine(_repoRoot, "artifacts", "tmp", _buildConfiguration.ToLowerInvariant());
227+
Directory.CreateDirectory(artifactsTemp);
228+
229+
string tempPath = Path.Combine(artifactsTemp, $"CoverletMTP_Test_{Guid.NewGuid():N}");
234230
Directory.CreateDirectory(tempPath);
235231

236232
// Create NuGet.config to use local packages
@@ -496,10 +492,28 @@ private async Task<TestResult> RunTestsWithCoverage(string projectPath, string a
496492
string projectName = Path.GetFileNameWithoutExtension(projectPath);
497493
string testExecutable = Path.Combine(projectDir, "bin", _buildConfiguration, _buildTargetFramework, $"{projectName}.dll");
498494

495+
if (!File.Exists(testExecutable))
496+
{
497+
throw new FileNotFoundException(
498+
$"Test executable not found: {testExecutable}\n" +
499+
$"Build may have failed silently.");
500+
}
501+
502+
string coverletMtpDll = Path.Combine(
503+
Path.GetDirectoryName(testExecutable)!,
504+
"coverlet.MTP.dll");
505+
506+
if (!File.Exists(coverletMtpDll))
507+
{
508+
throw new FileNotFoundException(
509+
$"Coverlet MTP extension not found: {coverletMtpDll}\n" +
510+
$"The coverlet.MTP NuGet package may not have restored correctly.");
511+
}
512+
499513
var processStartInfo = new ProcessStartInfo
500514
{
501515
FileName = "dotnet",
502-
Arguments = $"exec \"{testExecutable}\" {arguments}",
516+
Arguments = $"exec \"{testExecutable}\" {arguments} --diagnostic --diagnostic-verbosity trace",
503517
RedirectStandardOutput = true,
504518
RedirectStandardError = true,
505519
UseShellExecute = false,
@@ -514,12 +528,28 @@ private async Task<TestResult> RunTestsWithCoverage(string projectPath, string a
514528

515529
await process.WaitForExitAsync();
516530

531+
string errorContext = process.ExitCode switch
532+
{
533+
0 => "Success",
534+
1 => "Test failures occurred",
535+
2 => "Invalid command-line arguments",
536+
3 => "Test discovery failed",
537+
4 => "Test execution failed",
538+
5 => "Unexpected error (unhandled exception)",
539+
_ => "Unknown error"
540+
};
541+
517542
return new TestResult
518543
{
519544
ExitCode = process.ExitCode,
545+
ErrorText = errorContext,
520546
StandardOutput = output,
521547
StandardError = error,
522-
CombinedOutput = $"STDOUT:\n{output}\n\nSTDERR:\n{error}"
548+
CombinedOutput = $"=== TEST EXECUTABLE ===\n{testExecutable}\n\n" +
549+
$"=== ARGUMENTS ===\n{arguments}\n\n" +
550+
$"=== EXIT CODE ===\n{process.ExitCode}\n\n" +
551+
$"=== STDOUT ===\n{output}\n\n" +
552+
$"=== STDERR ===\n{error}"
523553
};
524554
}
525555

@@ -544,24 +574,43 @@ public TestProject(string projectPath, string outputDirectory)
544574

545575
public void Dispose()
546576
{
547-
try
577+
string? projectDir = Path.GetDirectoryName(ProjectPath);
578+
if (projectDir == null || !Directory.Exists(projectDir))
579+
return;
580+
581+
// Retry cleanup to handle file locks (especially on Windows)
582+
for (int i = 0; i < 3; i++)
548583
{
549-
string? projectDir = Path.GetDirectoryName(ProjectPath);
550-
if (projectDir != null && Directory.Exists(projectDir))
584+
try
551585
{
552-
Directory.Delete(projectDir, true);
586+
Directory.Delete(projectDir, recursive: true);
587+
return; // Success
588+
}
589+
catch (IOException) when (i < 2)
590+
{
591+
// File may be locked by antivirus or other process
592+
System.Threading.Thread.Sleep(100);
593+
}
594+
catch (UnauthorizedAccessException) when (i < 2)
595+
{
596+
// Mark files as normal (remove read-only) and retry
597+
foreach (var file in Directory.GetFiles(projectDir, "*", SearchOption.AllDirectories))
598+
{
599+
File.SetAttributes(file, FileAttributes.Normal);
600+
}
601+
System.Threading.Thread.Sleep(100);
553602
}
554603
}
555-
catch
556-
{
557-
// Swallow cleanup exceptions
558-
}
604+
605+
// Log cleanup failure but don't throw (test already finished)
606+
Debug.WriteLine($"Warning: Failed to cleanup test directory: {projectDir}");
559607
}
560608
}
561609

562610
private class TestResult
563611
{
564612
public int ExitCode { get; set; }
613+
public string ErrorText { get; set; } = string.Empty;
565614
public string StandardOutput { get; set; } = string.Empty;
566615
public string StandardError { get; set; } = string.Empty;
567616
public string CombinedOutput { get; set; } = string.Empty;

test/coverlet.MTP.validation.tests/HelpCommandTests.cs

Lines changed: 16 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -22,20 +22,21 @@ public class HelpCommandTests
2222
private const string PropsFileName = "MTPTest.props";
2323
private string[] _testProjectTfms = [];
2424
private static readonly string s_projectName = "coverlet.MTP.validation.tests";
25-
private static readonly string s_sutName = "BasicTestProject";
25+
private const string sutName = "BasicTestProject";
2626
private readonly string _projectOutputPath = TestUtils.GetTestBinaryPath(s_projectName);
2727
private readonly string _testProjectPath;
28+
private readonly string _repoRoot ;
2829

2930
public HelpCommandTests()
3031
{
3132
_buildConfiguration = "Debug";
3233
_buildTargetFramework = "net8.0";
3334

3435
// Get repository root
35-
string repoRoot = Path.GetFullPath(Path.Combine(Directory.GetCurrentDirectory(), "..", "..", "..", ".."));
36-
_localPackagesPath = Path.Combine(repoRoot, "artifacts", "packages", _buildConfiguration.ToLowerInvariant(), "Shipping");
36+
_repoRoot = Path.GetFullPath(Path.Combine(Directory.GetCurrentDirectory(), "..", "..", "..", ".."));
37+
_localPackagesPath = Path.Combine(_repoRoot, "artifacts", "packages", _buildConfiguration.ToLowerInvariant(), "Shipping");
3738

38-
_projectOutputPath = Path.Combine(repoRoot, "artifacts", "bin", s_projectName, _buildConfiguration.ToLowerInvariant());
39+
_projectOutputPath = Path.Combine(_repoRoot, "artifacts", "bin", s_projectName, _buildConfiguration.ToLowerInvariant());
3940

4041
// Use dedicated test project in TestProjects subdirectory
4142
_testProjectPath = Path.Combine(
@@ -83,7 +84,7 @@ private void CreateDeterministicTestPropsFile()
8384
new XElement("PropertyGroup",
8485
new XElement("coverletMTPVersion", GetPackageVersion("*MTP*.nupkg")))));
8586

86-
string csprojPath = Path.Combine(_testProjectPath, s_sutName + ".csproj");
87+
string csprojPath = Path.Combine(_testProjectPath, sutName + ".csproj");
8788
XElement csproj = XElement.Load(csprojPath)!;
8889

8990
// Use only the first top-level PropertyGroup in the project file
@@ -457,11 +458,7 @@ private async Task RestoreProject(string projectPath)
457458

458459
private void VerifyCoverletMtpDeployed()
459460
{
460-
string binPath = Path.Combine(
461-
_testProjectPath,
462-
"bin",
463-
_buildConfiguration,
464-
_buildTargetFramework);
461+
string binPath = GetSUTBinaryPath();
465462

466463
string coverletMtpDll = Path.Combine(binPath, "coverlet.MTP.dll");
467464
string coverletCoreDll = Path.Combine(binPath, "coverlet.core.dll");
@@ -481,6 +478,13 @@ private void VerifyCoverletMtpDeployed()
481478
}
482479
}
483480

481+
private string GetSUTBinaryPath()
482+
{
483+
string binTestProjectPath = Path.Combine(_repoRoot, "artifacts", "bin", sutName);
484+
string binPath = Path.Combine(binTestProjectPath, _buildConfiguration);
485+
return binPath;
486+
}
487+
484488
private void UpdateNuGetConfig()
485489
{
486490
string nugetConfigPath = Path.Combine(_testProjectPath, "NuGet.config");
@@ -534,12 +538,7 @@ private async Task<int> BuildProject(string projectPath)
534538

535539
private async Task<TestResult> RunTestsWithHelp()
536540
{
537-
string testExecutable = Path.Combine(
538-
_testProjectPath,
539-
"bin",
540-
_buildConfiguration,
541-
_buildTargetFramework,
542-
"BasicTestProject.dll");
541+
string testExecutable = Path.Combine(GetSUTBinaryPath(), sutName + ".dll");
543542

544543
var processStartInfo = new ProcessStartInfo
545544
{
@@ -570,12 +569,7 @@ private async Task<TestResult> RunTestsWithHelp()
570569

571570
private async Task<TestResult> RunTestsWithInfo()
572571
{
573-
string testExecutable = Path.Combine(
574-
_testProjectPath,
575-
"bin",
576-
_buildConfiguration,
577-
_buildTargetFramework,
578-
"BasicTestProject.dll");
572+
string testExecutable = Path.Combine(GetSUTBinaryPath(), sutName + ".dll");
579573

580574
var processStartInfo = new ProcessStartInfo
581575
{

test/coverlet.MTP.validation.tests/TestProjects/CalculateClassLibrary/Class1.cs renamed to test/coverlet.MTP.validation.tests/TestProjects/CalculateClassLibrary/CalculateClass.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33

44
namespace CalculateClassLibrary
55
{
6-
public class Class1
6+
public class CalculateClass
77
{
88
public static int Add(int x, int y) =>
99
x + y;

test/coverlet.MTP.validation.tests/TestProjects/CalculateClassLibrary/ClassLibrary.csproj renamed to test/coverlet.MTP.validation.tests/TestProjects/CalculateClassLibrary/CalculateClassLibrary.csproj

File renamed without changes.

0 commit comments

Comments
 (0)