Issue #1024: Make AutomaticBaselineSelectionIsCorrect test stable enough

Author: Serg046

tests/BenchmarkDotNet.IntegrationTests/AutomaticBaselineTests.cs

@@ -1,11 +1,9 @@
-using BenchmarkDotNet.Attributes;
-using BenchmarkDotNet.Columns;
+using BenchmarkDotNet.Columns;
 using BenchmarkDotNet.Configs;
-using BenchmarkDotNet.Jobs;
 using BenchmarkDotNet.Reports;
-using BenchmarkDotNet.Running;
+using BenchmarkDotNet.Tests.Mocks;
+using System;
 using System.Linq;
-using System.Threading;
 using Xunit;
 
 namespace BenchmarkDotNet.IntegrationTests
@@ -15,39 +13,21 @@ public class AutomaticBaselineTests
         [Fact]
         public void AutomaticBaselineSelectionIsCorrect()
         {
-            var config = CreateConfig();
-
-            var summary = BenchmarkRunner.Run<BaselineSample>();
+            var config = ManualConfig.CreateEmpty()
+                .AddColumnProvider(DefaultColumnProviders.Instance)
+                .WithAutomaticBaseline(AutomaticBaselineMode.Fastest);
 
+            var summary = MockFactory.CreateSummary(config, hugeSd: true, Array.Empty<Metric>());
             var table = summary.GetTable(SummaryStyle.Default);
-            var column = table.Columns.Single(c => c.Header == "Ratio");
-            Assert.Equal(2, column.Content.Length);
-            Assert.Equal(1.0, double.Parse(column.Content[1])); // Ratio of TwoMilliseconds
-            Assert.True(double.Parse(column.Content[0]) > 1.0); // Ratio of TwoHundredMilliseconds
-        }
+            var method = table.Columns.Single(c => c.Header == "Method");
+            var mean = table.Columns.Single(c => c.Header == "Mean");
+            var ratio = table.Columns.Single(c => c.Header == "Ratio");
 
-        [AutomaticBaseline(AutomaticBaselineMode.Fastest)]
-        public class BaselineSample
-        {
-            [Benchmark]
-            public void TwoHundredMilliseconds()
-            {
-                Thread.Sleep(200);
-            }
-
-            [Benchmark]
-            public void TwoMilliseconds()
-            {
-                Thread.Sleep(2);
-            }
+            Assert.Equal(2, method.Content.Length);
+            Assert.Equal(nameof(MockFactory.MockBenchmarkClass.Foo), method.Content[0]);
+            Assert.Equal(1.0, double.Parse(ratio.Content[0])); // A faster one, see measurements in MockFactory.cs
+            Assert.Equal(nameof(MockFactory.MockBenchmarkClass.Bar), method.Content[1]);
+            Assert.Equal(1.5, double.Parse(ratio.Content[1]));
         }
-
-        private IConfig CreateConfig()
-            => ManualConfig.CreateEmpty()
-                .AddJob(Job.ShortRun
-                    .WithEvaluateOverhead(false) // no need to run idle for this test
-                    .WithWarmupCount(0) // don't run warmup to save some time for our CI runs
-                    .WithIterationCount(1)) // single iteration is enough for us
-                .AddColumnProvider(DefaultColumnProviders.Instance);
     }
 }