Skip to content

Commit 40ff038

Browse files
karwakarwa
committed
Report os_signposts on Darwin, split warmup reporting
1 parent 8163295 commit 40ff038

File tree

5 files changed

+223
-14
lines changed

5 files changed

+223
-14
lines changed

‎Sources/Benchmark/BenchmarkReporter.swift

+72-10
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,19 @@
1414

1515
import Foundation
1616

17+
#if canImport(OSLog)
18+
import OSLog
19+
#endif
20+
1721
protocol ProgressReporter {
18-
mutating func report(running name: String, suite: String)
19-
mutating func report(finishedRunning name: String, suite: String, nanosTaken: UInt64)
22+
mutating func reportWillBeginBenchmark(_ benchmark: AnyBenchmark, suite: BenchmarkSuite)
23+
mutating func reportFinishedBenchmark(nanosTaken: UInt64)
24+
25+
mutating func reportWarmingUp()
26+
mutating func reportFinishedWarmup(nanosTaken: UInt64)
27+
28+
mutating func reportRunning()
29+
mutating func reportFinishedRunning(nanosTaken: UInt64)
2030
}
2131

2232
protocol BenchmarkReporter {
@@ -25,32 +35,84 @@ protocol BenchmarkReporter {
2535

2636
struct VerboseProgressReporter<Output: FlushableTextOutputStream>: ProgressReporter {
2737
var output: Output
38+
var currentBenchmarkQualifiedName: String
39+
#if canImport(OSLog)
40+
var currentOSLog: Any?
41+
#endif
2842

2943
init(output: Output) {
3044
self.output = output
45+
self.currentBenchmarkQualifiedName = ""
46+
self.currentOSLog = nil
3147
}
3248

33-
mutating func report(running name: String, suite: String) {
49+
mutating func reportWillBeginBenchmark(_ benchmark: AnyBenchmark, suite: BenchmarkSuite) {
50+
3451
let prefix: String
35-
if suite != "" {
36-
prefix = "\(suite): "
52+
if suite.name != "" {
53+
prefix = "\(suite.name): "
3754
} else {
3855
prefix = ""
3956
}
40-
print("running \(prefix)\(name)...", terminator: "", to: &output)
57+
self.currentBenchmarkQualifiedName = "\(prefix)\(benchmark.name)"
58+
#if canImport(OSLog)
59+
if #available(macOS 10.14, iOS 12.0, watchOS 5.0, tvOS 12.0, *) {
60+
currentOSLog = OSLog(
61+
subsystem: currentBenchmarkQualifiedName, category: .pointsOfInterest)
62+
}
63+
#endif
64+
}
65+
66+
mutating func reportWarmingUp() {
67+
print("Warming up... ", terminator: "", to: &output)
68+
output.flush()
69+
#if canImport(OSLog)
70+
if #available(macOS 10.14, iOS 12.0, watchOS 5.0, tvOS 12.0, *) {
71+
os_signpost(.begin, log: currentOSLog as! OSLog, name: "Warmup")
72+
}
73+
#endif
74+
}
75+
76+
mutating func reportFinishedWarmup(nanosTaken: UInt64) {
77+
#if canImport(OSLog)
78+
if #available(macOS 10.14, iOS 12.0, watchOS 5.0, tvOS 12.0, *) {
79+
os_signpost(.end, log: currentOSLog as! OSLog, name: "Warmup")
80+
}
81+
#endif
82+
}
83+
84+
mutating func reportRunning() {
85+
print("Running \(currentBenchmarkQualifiedName)... ", terminator: "", to: &output)
4186
output.flush()
87+
#if canImport(OSLog)
88+
if #available(macOS 10.14, iOS 12.0, watchOS 5.0, tvOS 12.0, *) {
89+
os_signpost(.begin, log: currentOSLog as! OSLog, name: "Benchmark")
90+
}
91+
#endif
92+
}
93+
94+
mutating func reportFinishedRunning(nanosTaken: UInt64) {
95+
#if canImport(OSLog)
96+
if #available(macOS 10.14, iOS 12.0, watchOS 5.0, tvOS 12.0, *) {
97+
os_signpost(.end, log: currentOSLog as! OSLog, name: "Benchmark")
98+
}
99+
#endif
42100
}
43101

44-
mutating func report(finishedRunning name: String, suite: String, nanosTaken: UInt64) {
102+
mutating func reportFinishedBenchmark(nanosTaken: UInt64) {
45103
let timeDuration = String(format: "%.2f ms", Float(nanosTaken) / 1000000.0)
46-
print(" done! (\(timeDuration))", to: &output)
104+
print("Done! (\(timeDuration))", to: &output)
47105
output.flush()
48106
}
49107
}
50108

51109
struct QuietReporter: ProgressReporter, BenchmarkReporter {
52-
mutating func report(running name: String, suite: String) {}
53-
mutating func report(finishedRunning name: String, suite: String, nanosTaken: UInt64) {}
110+
mutating func reportWillBeginBenchmark(_ benchmark: AnyBenchmark, suite: BenchmarkSuite) {}
111+
mutating func reportWarmingUp() {}
112+
mutating func reportFinishedWarmup(nanosTaken: UInt64) {}
113+
mutating func reportRunning() {}
114+
mutating func reportFinishedRunning(nanosTaken: UInt64) {}
115+
mutating func reportFinishedBenchmark(nanosTaken: UInt64) {}
54116
mutating func report(results: [BenchmarkResult]) {}
55117
}
56118

‎Sources/Benchmark/BenchmarkRunner.swift

+10-4
Original file line numberDiff line numberDiff line change
@@ -85,15 +85,21 @@ public struct BenchmarkRunner {
8585
return
8686
}
8787

88-
progress.report(running: benchmark.name, suite: suite.name)
88+
progress.reportWillBeginBenchmark(benchmark, suite: suite)
8989
let totalStart = now()
9090

9191
var warmupState: BenchmarkState? = nil
9292
if settings.warmupIterations > 0 {
93-
warmupState = doNIterations(
93+
progress.reportWarmingUp()
94+
let state = doNIterations(
9495
settings.warmupIterations, benchmark: benchmark, suite: suite, settings: settings)
96+
let warmupElapsed = state.endTime - state.startTime
97+
progress.reportFinishedWarmup(nanosTaken: warmupElapsed)
98+
warmupState = state
9599
}
96100

101+
progress.reportRunning()
102+
97103
var state: BenchmarkState
98104
if let n = settings.iterations {
99105
state = doNIterations(n, benchmark: benchmark, suite: suite, settings: settings)
@@ -105,8 +111,8 @@ public struct BenchmarkRunner {
105111
let totalEnd = now()
106112
let totalElapsed = totalEnd - totalStart
107113

108-
progress.report(
109-
finishedRunning: benchmark.name, suite: suite.name, nanosTaken: totalElapsed)
114+
progress.reportFinishedRunning(nanosTaken: state.endTime - state.startTime)
115+
progress.reportFinishedBenchmark(nanosTaken: totalElapsed)
110116

111117
let result = BenchmarkResult(
112118
benchmarkName: benchmark.name,

‎Tests/BenchmarkTests/CMakeLists.txt

+1
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ add_library(BenchmarkTests
77
BenchmarkSuiteTests.swift
88
CustomBenchmarkTests.swift
99
MockTextOutputStream.swift
10+
ProgressReporterTests.swift
1011
StatsTests.swift
1112
XCTTestManifests.swift)
1213

‎Tests/BenchmarkTests/ProgressReporterTests.swift

+139
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
// Copyright 2022 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
import XCTest
16+
17+
@testable import Benchmark
18+
19+
final class ProgressReporterTests: XCTestCase {
20+
21+
fileprivate var dummySuite: BenchmarkSuite {
22+
BenchmarkSuite(name: "SomeSuite") { suite in
23+
suite.benchmark("SomeBenchmark") { /* No-op */ }
24+
suite.benchmark("AnotherBenchmark") { /* No-op */ }
25+
}
26+
}
27+
28+
open class DummyProgressReporterBase: ProgressReporter {
29+
var didReportWillBegin = false
30+
var didReportWarmingUp = false
31+
var didReportFinishedWarmup = false
32+
var didReportRunning = false
33+
var didReportFinishedRunning = false
34+
var benchmarksFinished = 0
35+
36+
init() {}
37+
38+
private func resetCallbackFlags() {
39+
didReportWillBegin = false
40+
didReportWarmingUp = false
41+
didReportFinishedWarmup = false
42+
didReportRunning = false
43+
didReportFinishedRunning = false
44+
}
45+
46+
open func reportWillBeginBenchmark(_ benchmark: AnyBenchmark, suite: BenchmarkSuite) {
47+
resetCallbackFlags()
48+
didReportWillBegin = true
49+
}
50+
51+
open func reportFinishedBenchmark(nanosTaken: UInt64) {
52+
resetCallbackFlags()
53+
benchmarksFinished += 1
54+
}
55+
56+
open func reportWarmingUp() {
57+
didReportWarmingUp = true
58+
}
59+
60+
open func reportFinishedWarmup(nanosTaken: UInt64) {
61+
didReportFinishedWarmup = true
62+
}
63+
64+
open func reportRunning() {
65+
didReportRunning = true
66+
}
67+
68+
open func reportFinishedRunning(nanosTaken: UInt64) {
69+
didReportFinishedRunning = true
70+
}
71+
}
72+
73+
func testProgressReportNoWarmup() throws {
74+
75+
class Reporter: DummyProgressReporterBase {
76+
var remainingBenchmarks = ["SomeBenchmark", "AnotherBenchmark"]
77+
78+
override func reportWillBeginBenchmark(
79+
_ benchmark: AnyBenchmark, suite: BenchmarkSuite
80+
) {
81+
super.reportWillBeginBenchmark(benchmark, suite: suite)
82+
XCTAssertEqual(benchmark.name, remainingBenchmarks.removeFirst())
83+
XCTAssertEqual(suite.name, "SomeSuite")
84+
}
85+
override func reportFinishedBenchmark(nanosTaken: UInt64) {
86+
XCTAssertTrue(didReportWillBegin)
87+
XCTAssertFalse(didReportWarmingUp)
88+
XCTAssertFalse(didReportFinishedWarmup)
89+
XCTAssertTrue(didReportRunning)
90+
XCTAssertTrue(didReportFinishedRunning)
91+
super.reportFinishedBenchmark(nanosTaken: nanosTaken)
92+
}
93+
}
94+
95+
let reporter = Reporter()
96+
var runner = BenchmarkRunner(suites: [dummySuite], settings: [Iterations(5)])
97+
runner.progress = reporter
98+
try runner.run()
99+
XCTAssertEqual(reporter.remainingBenchmarks, [])
100+
XCTAssertEqual(reporter.benchmarksFinished, 2)
101+
}
102+
103+
func testProgressReportWithWarmup() throws {
104+
105+
class Reporter: DummyProgressReporterBase {
106+
var remainingBenchmarks = ["SomeBenchmark", "AnotherBenchmark"]
107+
108+
override func reportWillBeginBenchmark(
109+
_ benchmark: AnyBenchmark, suite: BenchmarkSuite
110+
) {
111+
super.reportWillBeginBenchmark(benchmark, suite: suite)
112+
XCTAssertEqual(benchmark.name, remainingBenchmarks.removeFirst())
113+
XCTAssertEqual(suite.name, "SomeSuite")
114+
}
115+
override func reportFinishedBenchmark(nanosTaken: UInt64) {
116+
XCTAssertTrue(didReportWillBegin)
117+
XCTAssertTrue(didReportWarmingUp)
118+
XCTAssertTrue(didReportFinishedWarmup)
119+
XCTAssertTrue(didReportRunning)
120+
XCTAssertTrue(didReportFinishedRunning)
121+
super.reportFinishedBenchmark(nanosTaken: nanosTaken)
122+
}
123+
}
124+
125+
let reporter = Reporter()
126+
var runner = BenchmarkRunner(
127+
suites: [dummySuite], settings: [WarmupIterations(42), Iterations(5)]
128+
)
129+
runner.progress = reporter
130+
try runner.run()
131+
XCTAssertEqual(reporter.remainingBenchmarks, [])
132+
XCTAssertEqual(reporter.benchmarksFinished, 2)
133+
}
134+
135+
static var allTests = [
136+
("testProgressReportNoWarmup", testProgressReportNoWarmup),
137+
("testProgressReportWithWarmup", testProgressReportWithWarmup),
138+
]
139+
}

‎Tests/BenchmarkTests/XCTTestManifests.swift

+1
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import XCTest
2424
testCase(BenchmarkSettingTests.allTests),
2525
testCase(BenchmarkSuiteTests.allTests),
2626
testCase(CustomBenchmarkTests.allTests),
27+
testCase(ProgressReporterTests.allTests),
2728
testCase(StatsTests.allTests),
2829
]
2930
}

0 commit comments

Comments
 (0)