Skip to content

Commit c6e818c

Browse files
BertkBertk
committed
Improve test reliability, diagnostics, and cleanup
- Enhance test output with detailed diagnostics and error context - Use robust assertions with informative failure messages - Create temp test projects under artifacts/tmp for isolation - Add retries and file attribute handling to test dir cleanup - Check for test executable and coverlet.MTP.dll before running - Refactor HelpCommandTests for consistent path handling - Fix sample class naming mismatch in test project - Add condition to MSBuild import for props file robustness - Update NuGet config and project file handling for clarity
1 parent caa817e commit c6e818c

File tree

8 files changed

+104
-58
lines changed

8 files changed

+104
-58
lines changed

‎src/coverlet.MTP/buildTransitive/coverlet.MTP.props‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
11
<Project>
2-
<Import Project="$(MSBuildThisFileDirectory)..\..\buildMultiTargeting\coverlet.MTP.props" />
2+
<Import Project="$(MSBuildThisFileDirectory)..\..\buildMultiTargeting\coverlet.MTP.props"
3+
Condition="Exists('$(MSBuildThisFileDirectory)..\..\buildMultiTargeting\coverlet.MTP.props')" />
34
</Project>

‎test/coverlet.MTP.validation.tests/CollectCoverageTests.cs‎

Lines changed: 84 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,9 @@
44
using System.Diagnostics;
55
using System.Text.Json;
66
using System.Xml.Linq;
7+
using Newtonsoft.Json.Serialization;
78
using Xunit;
9+
using Xunit.Sdk;
810

911
namespace coverlet.MTP.validation.tests;
1012

@@ -20,15 +22,16 @@ public class CollectCoverageTests
2022
private readonly string _localPackagesPath;
2123
private const string CoverageJsonFileName = "coverage.json";
2224
private const string CoverageCoberturaFileName = "coverage.cobertura.xml";
25+
private readonly string _repoRoot;
2326

2427
public CollectCoverageTests()
2528
{
2629
_buildConfiguration = "Debug";
2730
_buildTargetFramework = "net8.0";
2831

2932
// Get local packages path (adjust based on your build output)
30-
string repoRoot = Path.GetFullPath(Path.Combine(Directory.GetCurrentDirectory(), "..", "..", "..", ".."));
31-
_localPackagesPath = Path.Combine(repoRoot, "artifacts", "package", _buildConfiguration.ToLowerInvariant());
33+
_repoRoot = Path.GetFullPath(Path.Combine(Directory.GetCurrentDirectory(), "..", "..", "..", ".."));
34+
_localPackagesPath = Path.Combine(_repoRoot, "artifacts", "package", _buildConfiguration.ToLowerInvariant());
3235
}
3336

3437
[Fact]
@@ -41,12 +44,10 @@ public async Task BasicCoverage_CollectsDataForCoveredLines()
4144
// Act
4245
var result = await RunTestsWithCoverage(testProject.ProjectPath, "--coverage");
4346

44-
TestContext.Current.AddAttachment(
45-
"Test Output",
46-
result.CombinedOutput);
47+
TestContext.Current?.AddAttachment("Test Output", result.CombinedOutput);
4748

4849
// Assert
49-
Assert.Equal(0, result.ExitCode);
50+
Assert.True( result.ExitCode == 0, $"Expected successful test run (exit code 0) but got {result.ExitCode}.\n\n{result.CombinedOutput}");
5051
Assert.Contains("Passed!", result.StandardOutput);
5152

5253
string[] coverageFiles = Directory.GetFiles(testProject.OutputDirectory, CoverageJsonFileName, SearchOption.AllDirectories);
@@ -69,12 +70,10 @@ public async Task CoverageWithFormat_GeneratesCorrectOutputFormat()
6970
testProject.ProjectPath,
7071
"--coverage --coverage-output-format cobertura");
7172

72-
TestContext.Current.AddAttachment(
73-
"Test Output",
74-
result.CombinedOutput);
73+
TestContext.Current?.AddAttachment("Test Output", result.CombinedOutput);
7574

7675
// Assert
77-
Assert.Equal(0, result.ExitCode);
76+
Assert.True(result.ExitCode == 0, $"Expected successful test run (exit code 0) but got {result.ExitCode}.\n\n{result.CombinedOutput}");
7877

7978
string[] coverageFiles = Directory.GetFiles(testProject.OutputDirectory, CoverageCoberturaFileName, SearchOption.AllDirectories);
8079
Assert.NotEmpty(coverageFiles);
@@ -94,12 +93,10 @@ public async Task CoverageInstrumentation_TracksMethodHits()
9493
// Act
9594
var result = await RunTestsWithCoverage(testProject.ProjectPath, "--coverage");
9695

97-
TestContext.Current.AddAttachment(
98-
"Test Output",
99-
result.CombinedOutput);
96+
TestContext.Current?.AddAttachment("Test Output", result.CombinedOutput);
10097

10198
// Assert
102-
Assert.Equal(0, result.ExitCode);
99+
Assert.True(result.ExitCode == 0, $"Expected successful test run (exit code 0) but got {result.ExitCode}.\n\n{result.CombinedOutput}");
103100

104101
string[] coverageFiles = Directory.GetFiles(testProject.OutputDirectory, CoverageJsonFileName, SearchOption.AllDirectories);
105102
var coverageData = ParseCoverageJson(coverageFiles[0]);
@@ -149,12 +146,10 @@ public async Task BranchCoverage_TracksConditionalPaths()
149146
// Act
150147
var result = await RunTestsWithCoverage(testProject.ProjectPath, "--coverage");
151148

152-
TestContext.Current.AddAttachment(
153-
"Test Output",
154-
result.CombinedOutput);
149+
TestContext.Current?.AddAttachment("Test Output", result.CombinedOutput)
155150

156151
// Assert
157-
Assert.Equal(0, result.ExitCode);
152+
Assert.True(result.ExitCode == 0, $"Expected successful test run (exit code 0) but got {result.ExitCode}.\n\n{result.CombinedOutput}");
158153

159154
string[] coverageFiles = Directory.GetFiles(testProject.OutputDirectory, CoverageJsonFileName, SearchOption.AllDirectories);
160155
var coverageData = ParseCoverageJson(coverageFiles[0]);
@@ -207,12 +202,10 @@ public async Task MultipleCoverageFormats_GeneratesAllReports()
207202
testProject.ProjectPath,
208203
"--coverage --coverage-output-format json,cobertura,lcov");
209204

210-
TestContext.Current.AddAttachment(
211-
"Test Output",
212-
result.CombinedOutput);
205+
TestContext.Current?.AddAttachment("Test Output", result.CombinedOutput);
213206

214207
// Assert
215-
Assert.Equal(0, result.ExitCode);
208+
Assert.True(result.ExitCode == 0, $"Expected successful test run (exit code 0) but got {result.ExitCode}.\n\n{result.CombinedOutput}");
216209

217210
// Verify all formats are generated
218211
Assert.NotEmpty(Directory.GetFiles(testProject.OutputDirectory, "coverage.json", SearchOption.AllDirectories));
@@ -223,14 +216,19 @@ public async Task MultipleCoverageFormats_GeneratesAllReports()
223216
#region Helper Methods
224217

225218
private TestProject CreateTestProject(
219+
226220
bool includeSimpleTest = false,
227221
bool includeMethodTests = false,
228222
bool includeMultipleClasses = false,
229223
bool includeCalculatorTest = false,
230224
bool includeBranchTest = false,
231225
bool includeMultipleTests = false)
232226
{
233-
string tempPath = Path.Combine(Path.GetTempPath(), $"CoverletMTP_Test_{Guid.NewGuid():N}");
227+
// Use repository artifacts folder instead of user temp
228+
string artifactsTemp = Path.Combine(_repoRoot, "artifacts", "tmp", _buildConfiguration.ToLowerInvariant());
229+
Directory.CreateDirectory(artifactsTemp);
230+
231+
string tempPath = Path.Combine(artifactsTemp, $"CoverletMTP_Test_{Guid.NewGuid():N}");
234232
Directory.CreateDirectory(tempPath);
235233

236234
// Create NuGet.config to use local packages
@@ -496,10 +494,28 @@ private async Task<TestResult> RunTestsWithCoverage(string projectPath, string a
496494
string projectName = Path.GetFileNameWithoutExtension(projectPath);
497495
string testExecutable = Path.Combine(projectDir, "bin", _buildConfiguration, _buildTargetFramework, $"{projectName}.dll");
498496

497+
if (!File.Exists(testExecutable))
498+
{
499+
throw new FileNotFoundException(
500+
$"Test executable not found: {testExecutable}\n" +
501+
$"Build may have failed silently.");
502+
}
503+
504+
string coverletMtpDll = Path.Combine(
505+
Path.GetDirectoryName(testExecutable)!,
506+
"coverlet.MTP.dll");
507+
508+
if (!File.Exists(coverletMtpDll))
509+
{
510+
throw new FileNotFoundException(
511+
$"Coverlet MTP extension not found: {coverletMtpDll}\n" +
512+
$"The coverlet.MTP NuGet package may not have restored correctly.");
513+
}
514+
499515
var processStartInfo = new ProcessStartInfo
500516
{
501517
FileName = "dotnet",
502-
Arguments = $"exec \"{testExecutable}\" {arguments}",
518+
Arguments = $"exec \"{testExecutable}\" {arguments} --diagnostic --diagnostic-verbosity trace",
503519
RedirectStandardOutput = true,
504520
RedirectStandardError = true,
505521
UseShellExecute = false,
@@ -514,12 +530,28 @@ private async Task<TestResult> RunTestsWithCoverage(string projectPath, string a
514530

515531
await process.WaitForExitAsync();
516532

533+
string errorContext = process.ExitCode switch
534+
{
535+
0 => "Success",
536+
1 => "Test failures occurred",
537+
2 => "Invalid command-line arguments",
538+
3 => "Test discovery failed",
539+
4 => "Test execution failed",
540+
5 => "Unexpected error (unhandled exception)",
541+
_ => "Unknown error"
542+
};
543+
517544
return new TestResult
518545
{
519546
ExitCode = process.ExitCode,
547+
ErrorText = errorContext,
520548
StandardOutput = output,
521549
StandardError = error,
522-
CombinedOutput = $"STDOUT:\n{output}\n\nSTDERR:\n{error}"
550+
CombinedOutput = $"=== TEST EXECUTABLE ===\n{testExecutable}\n\n" +
551+
$"=== ARGUMENTS ===\n{arguments}\n\n" +
552+
$"=== EXIT CODE ===\n{process.ExitCode}\n\n" +
553+
$"=== STDOUT ===\n{output}\n\n" +
554+
$"=== STDERR ===\n{error}"
523555
};
524556
}
525557

@@ -544,24 +576,43 @@ public TestProject(string projectPath, string outputDirectory)
544576

545577
public void Dispose()
546578
{
547-
try
579+
string? projectDir = Path.GetDirectoryName(ProjectPath);
580+
if (projectDir == null || !Directory.Exists(projectDir))
581+
return;
582+
583+
// Retry cleanup to handle file locks (especially on Windows)
584+
for (int i = 0; i < 3; i++)
548585
{
549-
string? projectDir = Path.GetDirectoryName(ProjectPath);
550-
if (projectDir != null && Directory.Exists(projectDir))
586+
try
551587
{
552-
Directory.Delete(projectDir, true);
588+
Directory.Delete(projectDir, recursive: true);
589+
return; // Success
590+
}
591+
catch (IOException) when (i < 2)
592+
{
593+
// File may be locked by antivirus or other process
594+
System.Threading.Thread.Sleep(100);
595+
}
596+
catch (UnauthorizedAccessException) when (i < 2)
597+
{
598+
// Mark files as normal (remove read-only) and retry
599+
foreach (var file in Directory.GetFiles(projectDir, "*", SearchOption.AllDirectories))
600+
{
601+
File.SetAttributes(file, FileAttributes.Normal);
602+
}
603+
System.Threading.Thread.Sleep(100);
553604
}
554605
}
555-
catch
556-
{
557-
// Swallow cleanup exceptions
558-
}
606+
607+
// Log cleanup failure but don't throw (test already finished)
608+
Debug.WriteLine($"Warning: Failed to cleanup test directory: {projectDir}");
559609
}
560610
}
561611

562612
private class TestResult
563613
{
564614
public int ExitCode { get; set; }
615+
public string ErrorText { get; set; } = string.Empty;
565616
public string StandardOutput { get; set; } = string.Empty;
566617
public string StandardError { get; set; } = string.Empty;
567618
public string CombinedOutput { get; set; } = string.Empty;

‎test/coverlet.MTP.validation.tests/HelpCommandTests.cs‎

Lines changed: 16 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -22,20 +22,21 @@ public class HelpCommandTests
2222
private const string PropsFileName = "MTPTest.props";
2323
private string[] _testProjectTfms = [];
2424
private static readonly string s_projectName = "coverlet.MTP.validation.tests";
25-
private static readonly string s_sutName = "BasicTestProject";
25+
private const string _sutName = "BasicTestProject";
2626
private readonly string _projectOutputPath = TestUtils.GetTestBinaryPath(s_projectName);
2727
private readonly string _testProjectPath;
28+
private readonly string _repoRoot ;
2829

2930
public HelpCommandTests()
3031
{
3132
_buildConfiguration = "Debug";
3233
_buildTargetFramework = "net8.0";
3334

3435
// Get repository root
35-
string repoRoot = Path.GetFullPath(Path.Combine(Directory.GetCurrentDirectory(), "..", "..", "..", ".."));
36-
_localPackagesPath = Path.Combine(repoRoot, "artifacts", "packages", _buildConfiguration.ToLowerInvariant(), "Shipping");
36+
_repoRoot = Path.GetFullPath(Path.Combine(Directory.GetCurrentDirectory(), "..", "..", "..", ".."));
37+
_localPackagesPath = Path.Combine(_repoRoot, "artifacts", "packages", _buildConfiguration.ToLowerInvariant(), "Shipping");
3738

38-
_projectOutputPath = Path.Combine(repoRoot, "artifacts", "bin", s_projectName, _buildConfiguration.ToLowerInvariant());
39+
_projectOutputPath = Path.Combine(_repoRoot, "artifacts", "bin", s_projectName, _buildConfiguration.ToLowerInvariant());
3940

4041
// Use dedicated test project in TestProjects subdirectory
4142
_testProjectPath = Path.Combine(
@@ -83,7 +84,7 @@ private void CreateDeterministicTestPropsFile()
8384
new XElement("PropertyGroup",
8485
new XElement("coverletMTPVersion", GetPackageVersion("*MTP*.nupkg")))));
8586

86-
string csprojPath = Path.Combine(_testProjectPath, s_sutName + ".csproj");
87+
string csprojPath = Path.Combine(_testProjectPath, _sutName + ".csproj");
8788
XElement csproj = XElement.Load(csprojPath)!;
8889

8990
// Use only the first top-level PropertyGroup in the project file
@@ -457,11 +458,7 @@ private async Task RestoreProject(string projectPath)
457458

458459
private void VerifyCoverletMtpDeployed()
459460
{
460-
string binPath = Path.Combine(
461-
_testProjectPath,
462-
"bin",
463-
_buildConfiguration,
464-
_buildTargetFramework);
461+
string binPath = GetSUTBinaryPath();
465462

466463
string coverletMtpDll = Path.Combine(binPath, "coverlet.MTP.dll");
467464
string coverletCoreDll = Path.Combine(binPath, "coverlet.core.dll");
@@ -481,6 +478,13 @@ private void VerifyCoverletMtpDeployed()
481478
}
482479
}
483480

481+
private string GetSUTBinaryPath()
482+
{
483+
string binTestProjectPath = Path.Combine(_repoRoot, "artifacts", "bin", _sutName);
484+
string binPath = Path.Combine(binTestProjectPath, _buildConfiguration);
485+
return binPath;
486+
}
487+
484488
private void UpdateNuGetConfig()
485489
{
486490
string nugetConfigPath = Path.Combine(_testProjectPath, "NuGet.config");
@@ -534,12 +538,7 @@ private async Task<int> BuildProject(string projectPath)
534538

535539
private async Task<TestResult> RunTestsWithHelp()
536540
{
537-
string testExecutable = Path.Combine(
538-
_testProjectPath,
539-
"bin",
540-
_buildConfiguration,
541-
_buildTargetFramework,
542-
"BasicTestProject.dll");
541+
string testExecutable = Path.Combine(GetSUTBinaryPath(), _sutName + ".dll");
543542

544543
var processStartInfo = new ProcessStartInfo
545544
{
@@ -570,12 +569,7 @@ private async Task<TestResult> RunTestsWithHelp()
570569

571570
private async Task<TestResult> RunTestsWithInfo()
572571
{
573-
string testExecutable = Path.Combine(
574-
_testProjectPath,
575-
"bin",
576-
_buildConfiguration,
577-
_buildTargetFramework,
578-
"BasicTestProject.dll");
572+
string testExecutable = Path.Combine(GetSUTBinaryPath(), _sutName + ".dll");
579573

580574
var processStartInfo = new ProcessStartInfo
581575
{

test/coverlet.MTP.validation.tests/TestProjects/CalculateClassLibrary/Class1.cs renamed to test/coverlet.MTP.validation.tests/TestProjects/CalculateClassLibrary/CalculateClass.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33

44
namespace CalculateClassLibrary
55
{
6-
public class Class1
6+
public class CalculateClass
77
{
88
public static int Add(int x, int y) =>
99
x + y;

test/coverlet.MTP.validation.tests/TestProjects/CalculateClassLibrary/ClassLibrary.csproj renamed to test/coverlet.MTP.validation.tests/TestProjects/CalculateClassLibrary/CalculateClassLibrary.csproj

File renamed without changes.

test/coverlet.MTP.validation.tests/TestProjects/CalculateSampleTests/Tests.csproj renamed to test/coverlet.MTP.validation.tests/TestProjects/CalculateTestProject/CalculateTestProject.csproj

File renamed without changes.

test/coverlet.MTP.validation.tests/TestProjects/CalculateSampleTests/UnitTest1.cs renamed to test/coverlet.MTP.validation.tests/TestProjects/CalculateTestProject/UnitTest1.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,6 @@ public class UnitTest1
1111
[Fact]
1212
public void AddTest()
1313
{
14-
Assert.Equal(5, Class1.Add(2, 3));
14+
Assert.Equal(5, CalculateClass.Add(2, 3));
1515
}
1616
}

test/coverlet.MTP.validation.tests/TestProjects/CalculateSampleTests/xunit.runner.json renamed to test/coverlet.MTP.validation.tests/TestProjects/CalculateTestProject/xunit.runner.json

File renamed without changes.

0 commit comments

Comments
 (0)