Skip to content

Commit dbf38f8

Browse files
karwakarwa
committed
Report os_signposts on Darwin, split warmup reporting
1 parent 8163295 commit dbf38f8

File tree

5 files changed

+208
-12
lines changed

5 files changed

+208
-12
lines changed

‎Sources/Benchmark/BenchmarkReporter.swift

+65-9
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,18 @@
1414

1515
import Foundation
1616

17+
#if canImport(OSLog)
18+
import OSLog
19+
#endif
20+
1721
protocol ProgressReporter {
18-
mutating func report(running name: String, suite: String)
19-
mutating func report(finishedRunning name: String, suite: String, nanosTaken: UInt64)
22+
mutating func report(willBegin name: String, suite: String)
23+
24+
mutating func reportWarmingUp()
25+
mutating func reportFinishedWarmup(nanosTaken: UInt64)
26+
27+
mutating func reportRunning()
28+
mutating func reportFinishedRunning(nanosTaken: UInt64)
2029
}
2130

2231
protocol BenchmarkReporter {
@@ -25,32 +34,79 @@ protocol BenchmarkReporter {
2534

2635
struct VerboseProgressReporter<Output: FlushableTextOutputStream>: ProgressReporter {
2736
var output: Output
37+
var prefix: String
38+
var name: String
39+
#if canImport(OSLog)
40+
var currentOSLog: Any?
41+
#endif
2842

2943
init(output: Output) {
3044
self.output = output
45+
self.prefix = ""
46+
self.name = "<Benchmark Name>"
47+
self.currentOSLog = nil
3148
}
3249

33-
mutating func report(running name: String, suite: String) {
34-
let prefix: String
50+
mutating func report(willBegin name: String, suite: String) {
3551
if suite != "" {
3652
prefix = "\(suite): "
3753
} else {
3854
prefix = ""
3955
}
40-
print("running \(prefix)\(name)...", terminator: "", to: &output)
56+
self.name = name
57+
#if canImport(OSLog)
58+
if #available(macOS 10.14, iOS 12.0, watchOS 5.0, tvOS 12.0, *) {
59+
currentOSLog = OSLog(subsystem: "\(prefix)\(name)", category: .pointsOfInterest)
60+
}
61+
#endif
62+
}
63+
64+
mutating func reportWarmingUp() {
65+
print("Warming up... ", terminator: "", to: &output)
4166
output.flush()
67+
#if canImport(OSLog)
68+
if #available(macOS 10.14, iOS 12.0, watchOS 5.0, tvOS 12.0, *) {
69+
os_signpost(.begin, log: currentOSLog as! OSLog, name: "Warmup")
70+
}
71+
#endif
4272
}
4373

44-
mutating func report(finishedRunning name: String, suite: String, nanosTaken: UInt64) {
74+
mutating func reportFinishedWarmup(nanosTaken: UInt64) {
75+
#if canImport(OSLog)
76+
if #available(macOS 10.14, iOS 12.0, watchOS 5.0, tvOS 12.0, *) {
77+
os_signpost(.end, log: currentOSLog as! OSLog, name: "Warmup")
78+
}
79+
#endif
80+
}
81+
82+
mutating func reportRunning() {
83+
print("Running \(prefix)\(name)... ", terminator: "", to: &output)
84+
output.flush()
85+
#if canImport(OSLog)
86+
if #available(macOS 10.14, iOS 12.0, watchOS 5.0, tvOS 12.0, *) {
87+
os_signpost(.begin, log: currentOSLog as! OSLog, name: "Benchmark")
88+
}
89+
#endif
90+
}
91+
92+
mutating func reportFinishedRunning(nanosTaken: UInt64) {
4593
let timeDuration = String(format: "%.2f ms", Float(nanosTaken) / 1000000.0)
46-
print(" done! (\(timeDuration))", to: &output)
94+
print("Done! (\(timeDuration))", to: &output)
4795
output.flush()
96+
#if canImport(OSLog)
97+
if #available(macOS 10.14, iOS 12.0, watchOS 5.0, tvOS 12.0, *) {
98+
os_signpost(.end, log: currentOSLog as! OSLog, name: "Benchmark")
99+
}
100+
#endif
48101
}
49102
}
50103

51104
struct QuietReporter: ProgressReporter, BenchmarkReporter {
52-
mutating func report(running name: String, suite: String) {}
53-
mutating func report(finishedRunning name: String, suite: String, nanosTaken: UInt64) {}
105+
mutating func report(willBegin name: String, suite: String) {}
106+
mutating func reportWarmingUp() {}
107+
mutating func reportFinishedWarmup(nanosTaken: UInt64) {}
108+
mutating func reportRunning() {}
109+
mutating func reportFinishedRunning(nanosTaken: UInt64) {}
54110
mutating func report(results: [BenchmarkResult]) {}
55111
}
56112

‎Sources/Benchmark/BenchmarkRunner.swift

+7-3
Original file line numberDiff line numberDiff line change
@@ -85,15 +85,20 @@ public struct BenchmarkRunner {
8585
return
8686
}
8787

88-
progress.report(running: benchmark.name, suite: suite.name)
88+
progress.report(willBegin: benchmark.name, suite: suite.name)
8989
let totalStart = now()
9090

9191
var warmupState: BenchmarkState? = nil
9292
if settings.warmupIterations > 0 {
93+
progress.reportWarmingUp()
9394
warmupState = doNIterations(
9495
settings.warmupIterations, benchmark: benchmark, suite: suite, settings: settings)
96+
let warmupElapsed = warmupState!.endTime - warmupState!.startTime
97+
progress.reportFinishedWarmup(nanosTaken: warmupElapsed)
9598
}
9699

100+
progress.reportRunning()
101+
97102
var state: BenchmarkState
98103
if let n = settings.iterations {
99104
state = doNIterations(n, benchmark: benchmark, suite: suite, settings: settings)
@@ -105,8 +110,7 @@ public struct BenchmarkRunner {
105110
let totalEnd = now()
106111
let totalElapsed = totalEnd - totalStart
107112

108-
progress.report(
109-
finishedRunning: benchmark.name, suite: suite.name, nanosTaken: totalElapsed)
113+
progress.reportFinishedRunning(nanosTaken: totalElapsed)
110114

111115
let result = BenchmarkResult(
112116
benchmarkName: benchmark.name,

‎Tests/BenchmarkTests/CMakeLists.txt

+1
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ add_library(BenchmarkTests
77
BenchmarkSuiteTests.swift
88
CustomBenchmarkTests.swift
99
MockTextOutputStream.swift
10+
ProgressReporterTests.swift
1011
StatsTests.swift
1112
XCTTestManifests.swift)
1213

‎Tests/BenchmarkTests/ProgressReporterTests.swift

+134
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
// Copyright 2022 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
import XCTest
16+
17+
@testable import Benchmark
18+
19+
final class ProgressReporterTests: XCTestCase {
20+
21+
fileprivate var dummySuite: BenchmarkSuite {
22+
BenchmarkSuite(name: "SomeSuite") { suite in
23+
suite.benchmark("SomeBenchmark") { /* No-op */ }
24+
suite.benchmark("AnotherBenchmark") { /* No-op */ }
25+
}
26+
}
27+
28+
open class DummyProgressReporterBase: ProgressReporter {
29+
var didReportWillBegin = false
30+
var didReportWarmingUp = false
31+
var didReportFinishedWarmup = false
32+
var didReportRunning = false
33+
var didReportFinishedRunning = false
34+
35+
init() {}
36+
37+
private func resetCallbackFlags() {
38+
didReportWillBegin = false
39+
didReportWarmingUp = false
40+
didReportFinishedWarmup = false
41+
didReportRunning = false
42+
didReportFinishedRunning = false
43+
}
44+
45+
open func report(willBegin name: String, suite: String) {
46+
resetCallbackFlags()
47+
didReportWillBegin = true
48+
}
49+
50+
open func reportWarmingUp() {
51+
didReportWarmingUp = true
52+
}
53+
54+
open func reportFinishedWarmup(nanosTaken: UInt64) {
55+
didReportFinishedWarmup = true
56+
}
57+
58+
open func reportRunning() {
59+
didReportRunning = true
60+
}
61+
62+
open func reportFinishedRunning(nanosTaken: UInt64) {
63+
didReportFinishedRunning = true
64+
}
65+
}
66+
67+
func testProgressReportNoWarmup() throws {
68+
69+
class Reporter: DummyProgressReporterBase {
70+
var remainingBenchmarks = ["SomeBenchmark", "AnotherBenchmark"]
71+
72+
func checkFlags() {
73+
if didReportWillBegin {
74+
XCTAssertFalse(didReportWarmingUp)
75+
XCTAssertFalse(didReportFinishedWarmup)
76+
XCTAssertTrue(didReportRunning)
77+
XCTAssertTrue(didReportFinishedRunning)
78+
}
79+
}
80+
81+
override func report(willBegin name: String, suite: String) {
82+
// Check flags from the previous run, if there was one.
83+
checkFlags()
84+
// Reset the flags and check this run.
85+
super.report(willBegin: name, suite: suite)
86+
XCTAssertEqual(name, remainingBenchmarks.removeFirst())
87+
XCTAssertEqual(suite, "SomeSuite")
88+
}
89+
}
90+
91+
let reporter = Reporter()
92+
var runner = BenchmarkRunner(suites: [dummySuite], settings: [Iterations(5)])
93+
runner.progress = reporter
94+
try runner.run()
95+
reporter.checkFlags()
96+
}
97+
98+
func testProgressReportWithWarmup() throws {
99+
100+
class Reporter: DummyProgressReporterBase {
101+
var remainingBenchmarks = ["SomeBenchmark", "AnotherBenchmark"]
102+
103+
func checkFlags() {
104+
if didReportWillBegin {
105+
XCTAssertTrue(didReportWarmingUp)
106+
XCTAssertTrue(didReportFinishedWarmup)
107+
XCTAssertTrue(didReportRunning)
108+
XCTAssertTrue(didReportFinishedRunning)
109+
}
110+
}
111+
112+
override func report(willBegin name: String, suite: String) {
113+
// Check flags from the previous run, if there was one.
114+
checkFlags()
115+
// Reset the flags and check this run.
116+
super.report(willBegin: name, suite: suite)
117+
XCTAssertEqual(name, remainingBenchmarks.removeFirst())
118+
XCTAssertEqual(suite, "SomeSuite")
119+
}
120+
}
121+
122+
let reporter = Reporter()
123+
var runner = BenchmarkRunner(
124+
suites: [dummySuite], settings: [WarmupIterations(420), Iterations(5)])
125+
runner.progress = reporter
126+
try runner.run()
127+
reporter.checkFlags()
128+
}
129+
130+
static var allTests = [
131+
("testProgressReportNoWarmup", testProgressReportNoWarmup),
132+
("testProgressReportWithWarmup", testProgressReportWithWarmup),
133+
]
134+
}

‎Tests/BenchmarkTests/XCTTestManifests.swift

+1
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import XCTest
2424
testCase(BenchmarkSettingTests.allTests),
2525
testCase(BenchmarkSuiteTests.allTests),
2626
testCase(CustomBenchmarkTests.allTests),
27+
testCase(ProgressReporterTests.allTests),
2728
testCase(StatsTests.allTests),
2829
]
2930
}

0 commit comments

Comments
 (0)