Skip to content

Commit d672856

Browse files
filzrevfilzrev
committed
chore: add analyzer debug settings and sandbox project
1 parent bba26bc commit d672856

20 files changed

Lines changed: 595 additions & 0 deletions

‎BenchmarkDotNet.slnx‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
<Folder Name="/samples/">
77
<File Path="samples/Directory.Build.props" />
88
<Project Path="samples/BenchmarkDotNet.Samples.FSharp/BenchmarkDotNet.Samples.FSharp.fsproj" />
9+
<Project Path="samples/BenchmarkDotNet.Samples.Sandbox/BenchmarkDotNet.Samples.Sandbox.csproj" />
910
<Project Path="samples/BenchmarkDotNet.Samples/BenchmarkDotNet.Samples.csproj" DefaultStartup="true" />
1011
</Folder>
1112
<Folder Name="/src/">

‎samples/BenchmarkDotNet.Samples.Sandbox.slnf‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
{
2+
"solution": {
3+
"path": "..\\BenchmarkDotNet.slnx",
4+
"projects": [
5+
"samples\\BenchmarkDotNet.Samples.Sandbox\\BenchmarkDotNet.Samples.Sandbox.csproj",
6+
"src\\BenchmarkDotNet.Analyzers\\BenchmarkDotNet.Analyzers.csproj",
7+
"src\\BenchmarkDotNet.TestAdapter\\BenchmarkDotNet.TestAdapter.csproj",
8+
"src\\BenchmarkDotNet\\BenchmarkDotNet.csproj"
9+
]
10+
}
11+
}

‎samples/BenchmarkDotNet.Samples.Sandbox/.runsettings‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<RunSettings>
3+
<RunConfiguration>
4+
<EnvironmentVariables>
5+
<!-- Set `Debug` or `DebugInProcess` to when debugging benchmark -->
6+
<BENCHMARKDOTNET_CONFIG></BENCHMARKDOTNET_CONFIG>
7+
</EnvironmentVariables>
8+
</RunConfiguration>
9+
</RunSettings>

‎samples/BenchmarkDotNet.Samples.Sandbox/BenchmarkDotNet.Samples.Sandbox.csproj‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
<Import Project="..\..\build\common.props" />
3+
4+
<PropertyGroup>
5+
<OutputType>Exe</OutputType>
6+
<TargetFrameworks>net8.0;net9.0;net10.0</TargetFrameworks>
7+
<ImplicitUsings>enable</ImplicitUsings>
8+
<Nullable>enable</Nullable>
9+
</PropertyGroup>
10+
11+
<PropertyGroup Condition="$([MSBuild]::IsOSPlatform('Windows'))">
12+
<TargetFrameworks>$(TargetFrameworks);net48</TargetFrameworks>
13+
</PropertyGroup>
14+
15+
<PropertyGroup>
16+
<!-- Disable entry point generation as this project has it's own entry point -->
17+
<GenerateProgramFile>false</GenerateProgramFile>
18+
<!-- Disable parallel tests between TargetFrameworks -->
19+
<TestTfmsInParallel>false</TestTfmsInParallel>
20+
<!-- Enable .runsettings to use custom enviroment variable -->
21+
<RunSettingsFilePath>$(MSBuildProjectDirectory)\.runsettings</RunSettingsFilePath>
22+
</PropertyGroup>
23+
24+
<ItemGroup>
25+
<ProjectReference Include="..\..\src\BenchmarkDotNet\BenchmarkDotNet.csproj" />
26+
<ProjectReference Include="..\..\src\BenchmarkDotNet.Analyzers\BenchmarkDotNet.Analyzers.csproj">
27+
<OutputItemType>Analyzer</OutputItemType>
28+
<ReferenceOutputAssembly>false</ReferenceOutputAssembly>
29+
</ProjectReference>
30+
</ItemGroup>
31+
32+
<!-- Enable VS TestAdapter intergration -->
33+
<ItemGroup>
34+
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="18.0.1" />
35+
<ProjectReference Include="..\..\src\BenchmarkDotNet.TestAdapter\BenchmarkDotNet.TestAdapter.csproj" />
36+
</ItemGroup>
37+
38+
<Import Project="..\..\build\common.targets" />
39+
</Project>

‎samples/BenchmarkDotNet.Samples.Sandbox/BenchmarkDotNet/Configs/BaseBenchmarkConfig.cs‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
using BenchmarkDotNet.Attributes;
2+
using BenchmarkDotNet.Columns;
3+
using BenchmarkDotNet.Configs;
4+
using BenchmarkDotNet.Diagnosers;
5+
using BenchmarkDotNet.Exporters;
6+
using BenchmarkDotNet.Jobs;
7+
using BenchmarkDotNet.Loggers;
8+
using BenchmarkDotNet.Order;
9+
using BenchmarkDotNet.Reports;
10+
using Perfolizer.Horology;
11+
12+
namespace BenchmarkDotNet;
13+
14+
public class BaseBenchmarkConfig : ManualConfig
15+
{
16+
public BaseBenchmarkConfig()
17+
{
18+
WithSummaryStyle(SummaryStyle.Default.WithMaxParameterColumnWidth(40)); // Default: 20 chars
19+
WithBuildTimeout(TimeSpan.FromMinutes(10)); // Default: 120 seconds
20+
21+
WithOrderer(new DefaultOrderer());
22+
WithUnionRule(ConfigUnionRule.Union);
23+
WithArtifactsPath(DefaultConfig.Instance.ArtifactsPath);
24+
25+
#if DEBUG
26+
// Allow benchmarks for debug build.
27+
WithOptions(ConfigOptions.DisableOptimizationsValidator);
28+
#endif
29+
30+
// Enable following settings for debugging
31+
// WithOptions(ConfigOptions.StopOnFirstError);
32+
// WithOptions(ConfigOptions.KeepBenchmarkFiles);
33+
// WithOptions(ConfigOptions.GenerateMSBuildBinLog);
34+
}
35+
36+
// Use ShortRun based settings (LaunchCount=1 IterationCount=3 WarmupCount = 3)
37+
// And use RecommendedConfig setting that used by `dotnet/performance` repository.
38+
// https://github.com/dotnet/performance/blob/main/src/harness/BenchmarkDotNet.Extensions/RecommendedConfig.cs
39+
protected virtual Job GetBaseJobConfig() =>
40+
Job.Default
41+
.WithLaunchCount(1)
42+
.WithWarmupCount(3)
43+
.WithIterationTime(TimeInterval.FromMilliseconds(250)) // Default: 500 [ms]
44+
.WithMinIterationCount(15) // Default: 15
45+
.WithMaxIterationCount(20); // Default: 100
46+
47+
/// <summary>
48+
/// Add configurations.
49+
/// </summary>
50+
protected void AddConfigurations()
51+
{
52+
AddAnalyzers();
53+
AddColumnHidingRules();
54+
AddColumnProviders();
55+
AddDiagnosers();
56+
AddEventProcessors();
57+
AddExporters();
58+
AddFilters();
59+
AddHardwareCounters();
60+
AddLoggers();
61+
AddLogicalGroupRules();
62+
AddValidators();
63+
}
64+
65+
protected virtual void AddAnalyzers()
66+
{
67+
AddAnalyser(DefaultConfig.Instance.GetAnalysers().ToArray());
68+
}
69+
70+
protected virtual void AddColumnHidingRules()
71+
{
72+
}
73+
74+
protected virtual void AddColumnProviders()
75+
{
76+
AddColumnProvider(DefaultColumnProviders.Instance);
77+
}
78+
79+
protected virtual void AddDiagnosers()
80+
{
81+
AddDiagnoser(MemoryDiagnoser.Default);
82+
AddDiagnoser(new ExceptionDiagnoser(new ExceptionDiagnoserConfig(displayExceptionsIfZeroValue: false)));
83+
84+
#if NETCOREAPP3_0_OR_GREATER
85+
AddDiagnoser(new ThreadingDiagnoser(new ThreadingDiagnoserConfig(displayCompletedWorkItemCountWhenZero: false, displayLockContentionWhenZero: false)));
86+
#endif
87+
}
88+
89+
protected virtual void AddExporters()
90+
{
91+
// Use ConsoleMarkdownExporter to disable group higligting with `**`.
92+
AddExporter(MarkdownExporter.Console);
93+
}
94+
95+
protected virtual void AddEventProcessors()
96+
{
97+
}
98+
99+
protected virtual void AddFilters()
100+
{
101+
AddFilter(TargetFrameworkFilter.Instance);
102+
}
103+
104+
protected virtual void AddHardwareCounters()
105+
{
106+
}
107+
108+
protected virtual void AddLoggers()
109+
{
110+
AddLogger(ConsoleLogger.Default);
111+
}
112+
113+
protected virtual void AddLogicalGroupRules()
114+
{
115+
}
116+
117+
protected virtual void AddValidators()
118+
{
119+
AddValidator(DefaultConfig.Instance.GetValidators().ToArray());
120+
}
121+
}

‎samples/BenchmarkDotNet.Samples.Sandbox/BenchmarkDotNet/Configs/DebugBenchmarkConfig.cs‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
using BenchmarkDotNet.Configs;
2+
using BenchmarkDotNet.Engines;
3+
using BenchmarkDotNet.Jobs;
4+
using BenchmarkDotNet.Toolchains.InProcess.Emit;
5+
using System.Runtime.InteropServices;
6+
7+
namespace BenchmarkDotNet;
8+
9+
public class DebugBenchmarkConfig : BaseBenchmarkConfig
10+
{
11+
public DebugBenchmarkConfig()
12+
{
13+
// Configure base job config
14+
var baseJobConfig = GetBaseJobConfig();
15+
16+
// Add benchmark job.
17+
AddJob(baseJobConfig.WithCustomBuildConfiguration("Debug")
18+
.WithWarmupCount(1)
19+
.WithStrategy(RunStrategy.Monitoring)
20+
.WithId($"Debug({RuntimeInformation.FrameworkDescription})"));
21+
22+
// Set DebugConfig comatible option
23+
WithOptions(ConfigOptions.KeepBenchmarkFiles);
24+
25+
// Configure additional settings.
26+
AddConfigurations();
27+
}
28+
}

‎samples/BenchmarkDotNet.Samples.Sandbox/BenchmarkDotNet/Configs/DebugInProcessBenchmarkConfig.cs‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
using BenchmarkDotNet.Analysers;
2+
using BenchmarkDotNet.Configs;
3+
using BenchmarkDotNet.Engines;
4+
using BenchmarkDotNet.Jobs;
5+
using BenchmarkDotNet.Toolchains.InProcess.Emit;
6+
using System.Runtime.InteropServices;
7+
8+
namespace BenchmarkDotNet;
9+
10+
public class DebugInProcessBenchmarkConfig : BaseBenchmarkConfig
11+
{
12+
public DebugInProcessBenchmarkConfig() : base()
13+
{
14+
// Configure base job config
15+
var baseJobConfig = GetBaseJobConfig();
16+
17+
// Add benchmark job.
18+
AddJob(baseJobConfig.WithToolchain(InProcessEmitToolchain.Default)
19+
.WithWarmupCount(1)
20+
.WithStrategy(RunStrategy.Monitoring)
21+
.WithId($"DebugInProcess({RuntimeInformation.FrameworkDescription})"));
22+
23+
// Set DebugConfig comatible option
24+
WithOptions(ConfigOptions.KeepBenchmarkFiles);
25+
26+
// Configure additional settings.
27+
AddConfigurations();
28+
}
29+
30+
protected override void AddAnalyzers()
31+
{
32+
// Exclude MinIterationTimeAnalyser that cause warning when using RunStrategy.Monitoring.
33+
var analyzers = DefaultConfig.Instance
34+
.GetAnalysers()
35+
.Where(x => x is not MinIterationTimeAnalyser)
36+
.ToArray();
37+
38+
AddAnalyser(analyzers);
39+
}
40+
}

‎samples/BenchmarkDotNet.Samples.Sandbox/BenchmarkDotNet/Configs/DefaultBenchmarkConfig.cs‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
using BenchmarkDotNet.Jobs;
2+
using System.Runtime.InteropServices;
3+
4+
namespace BenchmarkDotNet;
5+
6+
public class DefaultBenchmarkConfig : BaseBenchmarkConfig
7+
{
8+
public DefaultBenchmarkConfig() : base()
9+
{
10+
// Configure base job config
11+
var baseJobConfig = GetBaseJobConfig();
12+
13+
// Create benchmark job.
14+
var job = baseJobConfig.WithId($"Default({RuntimeInformation.FrameworkDescription})");
15+
16+
// Add job.
17+
AddJob(job);
18+
19+
// Configure additional settings.
20+
AddConfigurations();
21+
}
22+
}

‎samples/BenchmarkDotNet.Samples.Sandbox/BenchmarkDotNet/Configs/TargetFrameworksBenchmarkConfig.cs‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
using BenchmarkDotNet.Analysers;
2+
using BenchmarkDotNet.Columns;
3+
using BenchmarkDotNet.Configs;
4+
using BenchmarkDotNet.Jobs;
5+
using BenchmarkDotNet.Toolchains.CsProj;
6+
7+
namespace BenchmarkDotNet;
8+
9+
/// <summary>
10+
/// BenchmarkConfig that run benchmarks for multiple target frameworks.
11+
/// </summary>
12+
public class TargetFrameworksBenchmarkConfig : BaseBenchmarkConfig
13+
{
14+
/// <summary>
15+
/// Initializes a new instance of the <see cref="DefaultBenchmarkConfig"/> class.
16+
/// </summary>
17+
public TargetFrameworksBenchmarkConfig() : base()
18+
{
19+
// Configure base job config
20+
var baseJobConfig = GetBaseJobConfig();
21+
22+
// Add jobs
23+
AddJob(baseJobConfig.WithToolchain(CsProjCoreToolchain.NetCoreApp80).WithId(".NET 8").AsBaseline());
24+
AddJob(baseJobConfig.WithToolchain(CsProjCoreToolchain.NetCoreApp90).WithId(".NET 9"));
25+
AddJob(baseJobConfig.WithToolchain(CsProjCoreToolchain.NetCoreApp10_0).WithId(".NET 10"));
26+
27+
// AddJob(baseJobConfig.WithToolchain(CsProjClassicNetToolchain.Net48).WithId(".NET Framework 4.8"));
28+
29+
// Configure additional settings.
30+
AddConfigurations();
31+
}
32+
33+
protected override void AddLogicalGroupRules()
34+
{
35+
// Grouping benchmarks by method.
36+
// Note: When following conditions are met. BaselineCustomAnalyzer raise warning. See: https://github.com/dotnet/BenchmarkDotNet/issues/2956
37+
// 1. Job contains Baseline=true
38+
// 2. Benchmark method contains Baseline=true
39+
// 3. Enable grouping with method.
40+
AddLogicalGroupRules(
41+
[
42+
BenchmarkLogicalGroupRule.ByMethod,
43+
]);
44+
}
45+
46+
protected override void AddColumnHidingRules()
47+
{
48+
HideColumns(Column.Toolchain); // Toolchain information are shown at JobId column.
49+
}
50+
}

‎samples/BenchmarkDotNet.Samples.Sandbox/BenchmarkDotNet/ExtensionMethods/SummariesExtensions.cs‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
using BenchmarkDotNet.Reports;
2+
3+
namespace BenchmarkDotNet;
4+
5+
public static class SummariesExtensions
6+
{
7+
public static bool HasError(this Summary[] summaries)
8+
{
9+
if (summaries.Length == 0)
10+
{
11+
var hashSet = new HashSet<string>(["--help", "--list", "--info", "--version"]);
12+
13+
var args = Environment.GetCommandLineArgs();
14+
return !args.Any(hashSet.Contains);
15+
}
16+
17+
if (summaries.Any(x => x.HasCriticalValidationErrors))
18+
return true;
19+
20+
return summaries.Any(x => x.Reports.Any(r => !r.Success));
21+
}
22+
}
23+

0 commit comments

Comments
 (0)