add benchmarks for coverlet.core and coverlet.msbuild

.github/workflows/benchmark.yml

@@ -0,0 +1,49 @@
+name: Build and run benchmarks
+
+# Description:
+# This workflow automates performance benchmarking for the Coverlet project.
+#
+# Triggers:
+# - Manual trigger via workflow_dispatch
+#
+# What it does:
+# 1. Sets up Ubuntu environment
+# 2. Installs .NET SDK based on global.json
+# 3. Builds benchmark tests in Release mode
+# 4. Runs performance benchmarks using BenchmarkDotNet
+# 5. Uploads benchmark results as artifacts
+#
+# Results:
+# - Benchmark artifacts are stored under: artifacts/bin/coverlet.core.benchmark.tests/release/BenchmarkDotNet.Artifacts/results/
+# - Results can be downloaded from GitHub Actions artifacts
+#
+# Usage:
+# - Can be manually triggered from Actions tab using workflow_dispatch
+# - Use results to monitor performance changes over time
+
+on:
+  workflow_dispatch:
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/[email protected]
+    - name: Setup .NET
+      uses: actions/[email protected]
+      with:
+        global-json-file: global.json
+        source-url: https://pkgs.dev.azure.com/bertk0374/_packaging/intern/nuget/v3/index.json
+    - name: Restore dependencies
+      run: dotnet restore
+    - name: Build release
+      run: dotnet build --no-restore -c Release ./test/coverlet.core.benchmark.tests/coverlet.core.benchmark.tests.csproj
+    - name: Run Benchmarks
+      run: cd ./artifacts/bin/coverlet.core.benchmark.tests/release && ./coverlet.core.benchmark.tests
+    - name: Upload benchmark results
+      uses: actions/upload-artifact@v2
+      with:
+        name: Benchmark_Results
+        path: ./artifacts/bin/coverlet.core.benchmark.tests/release/BenchmarkDotNet.Artifacts/results/*

.gitignore

@@ -315,6 +315,8 @@ coverage.*.cobertura.xml
 coverage.*.opencover.xml
 
 FolderProfile.pubxml
+BenchmarkDotNet.Artifacts/
 /NuGet.config
 nuget.config
 *.dmp
+test/coverlet.core.benchmark.tests/InstrumentationHelperBenchmarks.cs

Directory.Packages.props

@@ -17,6 +17,10 @@
     <XunitRunnerVisualstudioVersion>3.0.2</XunitRunnerVisualstudioVersion>
   </PropertyGroup>
   <ItemGroup>
+    <PackageVersion Include="BenchmarkDotNet" Version="0.14.0" />
+    <PackageVersion Include="BenchmarkDotNet.TestAdapter" Version="0.14.0" />
+    <PackageVersion Include="BenchmarkDotNet.Diagnostics.Windows" Version="0.14.0" />
+    <!--<PackageVersion Include="Microsoft.VisualStudio.DiagnosticsHub.BenchmarkDotNetDiagnosers" Version="17.13.35606.1" />-->
     <PackageVersion Include="DotNetConfig" Version="1.2.0" />
     <PackageVersion Include="Microsoft.Bcl.AsyncInterfaces" Version="8.0.0" />
     <PackageVersion Include="Microsoft.Build.Utilities.Core" Version="$(MicrosoftBuildVersion)" />

coverlet.sln

@@ -86,6 +86,8 @@ Project("{778DAE3C-4631-46EA-AA77-85C1314464D9}") = "coverlet.tests.projectsampl
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "coverlet.tests.utils", "test\coverlet.tests.utils\coverlet.tests.utils.csproj", "{0B109210-03CB-413F-888C-3023994AA384}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "coverlet.core.benchmark.tests", "test\coverlet.core.benchmark.tests\coverlet.core.benchmark.tests.csproj", "{8E8C4799-6F9D-49D8-96EA-E9BD1D187DAD}"
+EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "coverlet.tests.projectsample.wpf8.selfcontained", "test\coverlet.tests.projectsample.wpf8.selfcontained\coverlet.tests.projectsample.wpf8.selfcontained.csproj", "{71004336-9896-4AE5-8367-B29BB1680542}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "coverlet.core.coverage.tests", "test\coverlet.core.coverage.tests\coverlet.core.coverage.tests.csproj", "{F74AD549-EFE0-4CD9-AD10-B2189E3FD5BB}"
@@ -188,6 +190,10 @@ Global
 		{0B109210-03CB-413F-888C-3023994AA384}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{0B109210-03CB-413F-888C-3023994AA384}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{0B109210-03CB-413F-888C-3023994AA384}.Release|Any CPU.Build.0 = Release|Any CPU
+		{8E8C4799-6F9D-49D8-96EA-E9BD1D187DAD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{8E8C4799-6F9D-49D8-96EA-E9BD1D187DAD}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{8E8C4799-6F9D-49D8-96EA-E9BD1D187DAD}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{8E8C4799-6F9D-49D8-96EA-E9BD1D187DAD}.Release|Any CPU.Build.0 = Release|Any CPU
 		{71004336-9896-4AE5-8367-B29BB1680542}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{71004336-9896-4AE5-8367-B29BB1680542}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{71004336-9896-4AE5-8367-B29BB1680542}.Release|Any CPU.ActiveCfg = Release|Any CPU
@@ -225,6 +231,7 @@ Global
 		{351A034E-E642-4DB9-A21D-F71C8151C243} = {2FEBDE1B-83E3-445B-B9F8-5644B0E0E134}
 		{03400776-1F9A-4326-B927-1CA9B64B42A1} = {2FEBDE1B-83E3-445B-B9F8-5644B0E0E134}
 		{0B109210-03CB-413F-888C-3023994AA384} = {2FEBDE1B-83E3-445B-B9F8-5644B0E0E134}
+		{8E8C4799-6F9D-49D8-96EA-E9BD1D187DAD} = {2FEBDE1B-83E3-445B-B9F8-5644B0E0E134}
 		{71004336-9896-4AE5-8367-B29BB1680542} = {2FEBDE1B-83E3-445B-B9F8-5644B0E0E134}
 		{F74AD549-EFE0-4CD9-AD10-B2189E3FD5BB} = {2FEBDE1B-83E3-445B-B9F8-5644B0E0E134}
 	EndGlobalSection

src/coverlet.core/Properties/AssemblyInfo.cs

@@ -12,6 +12,7 @@
 
 [assembly: InternalsVisibleTo("coverlet.core.tests, PublicKey=0024000004800000940000000602000000240000525341310004000001000100757cf9291d78a82e5bb58a827a3c46c2f959318327ad30d1b52e918321ffbd847fb21565b8576d2a3a24562a93e86c77a298b564a0f1b98f63d7a1441a3a8bcc206da3ed09d5dacc76e122a109a9d3ac608e21a054d667a2bae98510a1f0f653c0e6f58f42b4b3934f6012f5ec4a09b3dfd3e14d437ede1424bdb722aead64ad")]
 [assembly: InternalsVisibleTo("coverlet.core.coverage.tests, PublicKey=0024000004800000940000000602000000240000525341310004000001000100094aad8eb75c06c9f2443dda84573b8db55cd6678452a60010db2643467ac28928db3a06b0b1ac3016645b448937d5e671b36504bcfc0fda27e996c5e1b0ee49747145cda6d47508d1e3c60b144634d95e33d4efe49536372df8139f48d3d897ae6931c2876d4f5d00215fd991cbcecde2705e53e19309e21c8b59d19eb925b1")]
+[assembly: InternalsVisibleTo("coverlet.core.benchmark.tests, PublicKey=002400000480000094000000060200000024000052534131000400000100010061d9d48f9cd6a4733ea1d88bc8a09c53a3040c3446c41858781df135170e8fe4e82a6cc6d9836f070ae0a28ebd7cd6e30dc1a853b350ae08ae77f437bc9f9f3b0ef23eb9b05eea38f97edb26a2dd2d0d8b32c6335c47b32f5277621118267f1a5717233eae25a3fe126d89d14b85a7a8e07657bf681a8a82100762a42ec477aa")]
 [assembly: InternalsVisibleTo("coverlet.collector.tests, PublicKey=0024000004800000940000000602000000240000525341310004000001000100ed0ed6af9693182615b8dcadc83c918b8d36312f86cefc69539d67d4189cd1b89420e7c3871802ffef7f5ca7816c68ad856c77bf7c230cc07824d96aa5d1237eebd30e246b9a14e22695fb26b40c800f74ea96619092cbd3a5d430d6c003fc7a82e8ccd1e315b935105d9232fe9e99e8d7ff54bba6f191959338d4a3169df9b3")]
 [assembly: InternalsVisibleTo("coverlet.msbuild.tasks.tests, PublicKey=002400000480000094000000060200000024000052534131000400000100010071b1583d63637a225f3f640252fee7130f0f3f2127d75025c1c3ee2d6dfc79a4950919268e0784d7ff54b0eadd8e4762e3e150da422e20e091eb0811d9d84e1779d5b95e349d5428aebb16e82e081bdf805926c5a9eb2094aaed9d36442de024264976a8835c7d6923047cf2f745e8f0ded2332f8980acd390f725224d976ed8")]
 [assembly: InternalsVisibleTo("coverlet.integration.tests, PublicKey=002400000480000094000000060200000024000052534131000400000100010001d24efbe9cbc2dc49b7a3d2ae34ca37cfb69b4f450acd768a22ce5cd021c8a38ae7dc68b2809a1ac606ad531b578f192a5690b2986990cbda4dd84ec65a3a4c1c36f6d7bb18f08592b93091535eaee2f0c8e48763ed7f190db2008e1f9e0facd5c0df5aaab74febd3430e09a428a72e5e6b88357f92d78e47512d46ebdc3cbb")]

test/coverlet.core.benchmark.tests/CoverageBenchmarks.cs

@@ -0,0 +1,68 @@
+// Copyright (c) Toni Solarin-Sodara
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System;
+using System.IO;
+using BenchmarkDotNet.Attributes;
+using Coverlet.Core;
+using Coverlet.Core.Abstractions;
+using Coverlet.Core.Helpers;
+using Coverlet.Core.Symbols;
+using Moq;
+
+namespace coverlet.core.benchmark.tests
+{
+  [MemoryDiagnoser]
+  public class CoverageBenchmarks
+  {
+    private Coverage _coverage;
+    private readonly Mock<ILogger> _mockLogger = new();
+    private DirectoryInfo _directory;
+
+    [GlobalSetup(Target = nameof(GetCoverageBenchmark))]
+    public void GetCoverageBenchmarkSetup()
+    {
+      string module = GetType().Assembly.Location;
+      string pdb = Path.Combine(Path.GetDirectoryName(module), Path.GetFileNameWithoutExtension(module) + ".pdb");
+
+      _directory = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()));
+
+      File.Copy(module, Path.Combine(_directory.FullName, Path.GetFileName(module)), true);
+      File.Copy(pdb, Path.Combine(_directory.FullName, Path.GetFileName(pdb)), true);
+
+      // TODO: Find a way to mimick hits
+      var instrumentationHelper =
+          new InstrumentationHelper(new ProcessExitHandler(), new RetryHelper(), new FileSystem(), new Mock<ILogger>().Object,
+                                    new SourceRootTranslator(module, new Mock<ILogger>().Object, new FileSystem(), new AssemblyAdapter()));
+
+      var parameters = new CoverageParameters
+      {
+        IncludeFilters = new string[] { "[coverlet.tests.projectsample.excludedbyattribute*]*" },
+        IncludeDirectories = Array.Empty<string>(),
+        ExcludeFilters = Array.Empty<string>(),
+        ExcludedSourceFiles = Array.Empty<string>(),
+        ExcludeAttributes = Array.Empty<string>(),
+        IncludeTestAssembly = false,
+        SingleHit = false,
+        MergeWith = string.Empty,
+        UseSourceLink = false
+      };
+
+      _coverage = new Coverage(Path.Combine(_directory.FullName, Path.GetFileName(module)), parameters, _mockLogger.Object, instrumentationHelper, new FileSystem(), new SourceRootTranslator(_mockLogger.Object, new FileSystem()), new CecilSymbolHelper());
+      _coverage.PrepareModules();
+
+    }
+
+    [GlobalCleanup]
+    public void IterationCleanup()
+    {
+      _directory.Delete(true);
+    }
+
+    [Benchmark]
+    public void GetCoverageBenchmark()
+    {
+      CoverageResult result = _coverage.GetCoverageResult();
+    }
+  }
+}

test/coverlet.core.benchmark.tests/HowTo.md

@@ -0,0 +1,134 @@
+# How to benchmark coverlet.core
+
+Coverlet.core.benchmark uses [BenchmarkDotNet](https://github.com/dotnet/BenchmarkDotNet) which has some runtime requirements
+
+- Build the project in `Release` mode
+- Make sure you have the latest version of the .NET SDK installed
+- Make sure you have the latest version of the BenchmarkDotNet package installed
+
+Use a terminal and run the following commands:
+
+```bash
+dotnet build test/coverlet.core.benchmark.tests -c release
+cd artifacts/bin/coverlet.core.benchmark.tests/release
+coverlet.core.benchmark.tests.exe
+```
+> [!TIP]
+> If error occurred missing `TestAssets\System.Private.CoreLib.dll` or `TestAssets\System.Private.CoreLib.pdb`.
+> Just copy the files from `artifacts\bin\coverlet.core.tests\debug\TestAssets`.
+
+The benchmark will automatically create reports in folder `BenchmarkDotNet.Artifacts` eg. find these files:
+
+```
+BenchmarkRun-20250411-083105.log
+results\BenchmarkRun-joined-2025-04-11-08-38-13-report-github.md
+results\BenchmarkRun-joined-2025-04-11-08-38-13-report.csv
+results\BenchmarkRun-joined-2025-04-11-08-38-13-report.html
+results\BenchmarkRun-joined-2025-04-11-08-55-34-report-github.md
+results\BenchmarkRun-joined-2025-04-11-08-55-34-report.csv
+results\BenchmarkRun-joined-2025-04-11-08-55-34-report.html
+```
+
+> [!NOTE]
+> This should be done for every coverlet release to avoid performance degradations.
+
+
+## Additional information
+- [BenchmarkDotNet](https://benchmarkdotnet.org)
+- [Analyze BenchmarkDotNet data in Visual Studio](https://learn.microsoft.com/en-us/visualstudio/profiling/profiling-with-benchmark-dotnet)
+- [.NET benchmarking and profiling for beginners](https://medium.com/ingeniouslysimple/net-benchmarking-and-profiling-for-beginners-62462e1e9a19)
+
+<blockquote>
+
+## Coverlet 6.0.0
+
+```
+
+BenchmarkDotNet v0.14.0, Windows 11 (10.0.26120.3671)
+AMD Ryzen 7 Microsoft Surface Edition, 1 CPU, 16 logical and 8 physical cores
+.NET SDK 9.0.203
+  [Host] : .NET 6.0.36 (6.0.3624.51421), X64 RyuJIT AVX2
+
+Job=ShortRun  Toolchain=InProcessNoEmitToolchain  IterationCount=3  
+LaunchCount=1  WarmupCount=3  
+
+```
+| Type                   | Method                | Mean                | Error              | StdDev            | Gen0        | Gen1        | Gen2      | Allocated    |
+|----------------------- |---------------------- |--------------------:|-------------------:|------------------:|------------:|------------:|----------:|-------------:|
+| CoverageBenchmarks     | GetCoverageBenchmark  |            46.42 ns |           1.670 ns |          0.092 ns |      0.0612 |           - |         - |        128 B |
+| InstrumenterBenchmarks | InstrumenterBenchmark | 4,938,713,766.67 ns | 767,760,955.034 ns | 42,083,568.809 ns | 857000.0000 | 109000.0000 | 2000.0000 | 2879633880 B |
+
+## Coverlet 6.0.1
+
+```
+
+BenchmarkDotNet v0.14.0, Windows 11 (10.0.26120.3671)
+AMD Ryzen 7 Microsoft Surface Edition, 1 CPU, 16 logical and 8 physical cores
+.NET SDK 8.0.408
+  [Host] : .NET 8.0.15 (8.0.1525.16413), X64 RyuJIT AVX2
+
+Job=ShortRun  Toolchain=InProcessNoEmitToolchain  IterationCount=3  
+LaunchCount=1  WarmupCount=3  
+
+```
+| Type                   | Method                | Mean                | Error              | StdDev            | Gen0        | Gen1       | Gen2      | Allocated    |
+|----------------------- |---------------------- |--------------------:|-------------------:|------------------:|------------:|-----------:|----------:|-------------:|
+| CoverageBenchmarks     | GetCoverageBenchmark  |            48.14 ns |           8.681 ns |          0.476 ns |      0.0612 |          - |         - |        128 B |
+| InstrumenterBenchmarks | InstrumenterBenchmark | 3,675,771,933.33 ns | 874,256,026.013 ns | 47,920,923.025 ns | 789000.0000 | 97000.0000 | 2000.0000 | 2864466608 B |
+
+## Coverlet 6.0.2
+
+```
+
+BenchmarkDotNet v0.14.0, Windows 11 (10.0.26120.3671)
+AMD Ryzen 7 Microsoft Surface Edition, 1 CPU, 16 logical and 8 physical cores
+.NET SDK 8.0.408
+  [Host] : .NET 8.0.15 (8.0.1525.16413), X64 RyuJIT AVX2
+
+Job=ShortRun  Toolchain=InProcessNoEmitToolchain  IterationCount=3  
+LaunchCount=1  WarmupCount=3  
+
+```
+| Type                   | Method                | Mean                 | Error               | StdDev             | Gen0        | Gen1        | Gen2      | Allocated    |
+|----------------------- |---------------------- |---------------------:|--------------------:|-------------------:|------------:|------------:|----------:|-------------:|
+| CoverageBenchmarks     | GetCoverageBenchmark  |             46.20 ns |            12.15 ns |           0.666 ns |      0.0612 |           - |         - |        128 B |
+| InstrumenterBenchmarks | InstrumenterBenchmark | 19,105,224,033.33 ns | 4,450,103,671.99 ns | 243,925,199.451 ns | 867000.0000 | 130000.0000 | 2000.0000 | 3170097400 B |
+
+## Coverlet 6.0.3
+
+```
+
+BenchmarkDotNet v0.14.0, Windows 11 (10.0.26120.3671)
+AMD Ryzen 7 Microsoft Surface Edition, 1 CPU, 16 logical and 8 physical cores
+.NET SDK 8.0.408
+  [Host] : .NET 8.0.15 (8.0.1525.16413), X64 RyuJIT AVX2
+
+Job=ShortRun  Toolchain=InProcessNoEmitToolchain  IterationCount=3  
+LaunchCount=1  WarmupCount=3  
+
+```
+| Type                   | Method                | Mean                | Error              | StdDev            | Gen0        | Gen1       | Gen2      | Allocated    |
+|----------------------- |---------------------- |--------------------:|-------------------:|------------------:|------------:|-----------:|----------:|-------------:|
+| CoverageBenchmarks     | GetCoverageBenchmark  |            47.32 ns |           4.246 ns |          0.233 ns |      0.0612 |          - |         - |        128 B |
+| InstrumenterBenchmarks | InstrumenterBenchmark | 3,620,665,600.00 ns | 580,611,812.738 ns | 31,825,292.772 ns | 775000.0000 | 91000.0000 | 2000.0000 | 2798558288 B |
+
+## Coverlet 6.0.4
+
+```
+
+BenchmarkDotNet v0.14.0, Windows 11 (10.0.26120.3671)
+AMD Ryzen 7 Microsoft Surface Edition, 1 CPU, 16 logical and 8 physical cores
+.NET SDK 8.0.408
+  [Host] : .NET 8.0.15 (8.0.1525.16413), X64 RyuJIT AVX2
+
+Job=ShortRun  Toolchain=InProcessNoEmitToolchain  IterationCount=3  
+LaunchCount=1  WarmupCount=3  
+
+```
+| Type                   | Method                | Mean                | Error              | StdDev            | Gen0        | Gen1       | Gen2      | Allocated    |
+|----------------------- |---------------------- |--------------------:|-------------------:|------------------:|------------:|-----------:|----------:|-------------:|
+| CoverageBenchmarks     | GetCoverageBenchmark  |            43.59 ns |           9.890 ns |          0.542 ns |      0.0612 |          - |         - |        128 B |
+| InstrumenterBenchmarks | InstrumenterBenchmark | 3,594,263,533.33 ns | 193,126,202.387 ns | 10,585,898.871 ns | 776000.0000 | 97000.0000 | 2000.0000 | 2798557560 B |
+
+
+</blockquote>