Use PkgBenchmark to detect performance regressions for heaps

.gitignore

@@ -3,3 +3,4 @@ docs/build/
 docs/site/
 docs/Manifest.toml
 Manifest.toml
+benchmark/*.json

.travis.yml

@@ -22,6 +22,23 @@ after_success:
 # https://juliadocs.github.io/Documenter.jl/stable/man/hosting/#.travis.yml-Configuration-1
 jobs:
   include:
+    - name: "Benchmark"
+      julia: 1.2
+      os: linux
+      before_script:
+        - git fetch origin '+refs/heads/master:refs/remotes/origin/master'
+        - git branch baseline origin/master
+        # Run benchmark outside `script` so that it's hidden by default:
+        - julia --project=benchmark -e '
+              using Pkg; Pkg.instantiate();
+              include("benchmark/runjudge.jl");'
+
+      script:
+        - julia --project=benchmark -e '
+              using Pkg; Pkg.instantiate();
+              include("benchmark/pprintjudge.jl");'
+      after_success: skip
+      if: type = pull_request
     - stage: "Documentation"
       julia: 1.2
       os: linux

benchmark/Project.toml

@@ -0,0 +1,3 @@
+[deps]
+BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
+PkgBenchmark = "32113eaa-f34f-5b0d-bd6c-c81e245fc73d"

benchmark/README.md

@@ -0,0 +1,30 @@
+## Notes on regression testing with PkgBenchmark
+
+### To troubleshoot benchmark script locally:
+```
+julia --project=benchmark -e '
+    using Pkg; Pkg.instantiate();
+    include("benchmark/runbenchmarks.jl");'
+```
+
+### To compare against baseline locally:
+
+Note, must have a `baseline` branch, which will be the refrence point against the currently active branch. A common use case is to point the baseline to the previous commit.
+
+This can be accomplished with
+```
+git branch baseline HEAD~
+```
+
+If there are errors preventing branch creation (likely due to earlier local benchmarking), may force a repoint with
+```
+git branch -f baseline HEAD~
+```
+
+Then run this command:
+```
+julia --project=benchmark -e '
+    using Pkg; Pkg.instantiate();
+    include("benchmark/runjudge.jl");
+    include("benchmark/pprintjudge.jl");'
+```

benchmark/benchmarks.jl

@@ -0,0 +1,41 @@
+using Pkg
+tempdir = mktempdir()
+Pkg.activate(tempdir)
+Pkg.develop(PackageSpec(path=joinpath(@__DIR__, "..")))
+Pkg.add(["BenchmarkTools", "PkgBenchmark", "Random"])
+Pkg.resolve()
+
+using DataStructures
+using BenchmarkTools
+using Random
+
+function push_heap(h::AbstractHeap, xs::Vector{Float64})
+    n = length(xs)
+
+    for i = 1 : n
+        push!(h, xs[i])
+    end
+end
+
+function pop_heap(h::AbstractHeap)
+    n = length(h)
+
+    for i = 1 : n
+        pop!(h)
+    end
+end
+
+Random.seed!(0)
+xs = rand(10^6)
+
+const SUITE = BenchmarkGroup()
+
+SUITE["heap"] = BenchmarkGroup(["binaryheap"])
+SUITE[["heap","basic", "min", "push"]] =
+    @benchmarkable push_heap(h, $xs) setup=(h=BinaryMinHeap{Float64}())
+SUITE[["heap","basic", "min", "pop"]] =
+    @benchmarkable pop_heap(h) setup=(h=BinaryMinHeap{Float64}($xs))
+SUITE[["heap","mutable", "min", "push"]] =
+    @benchmarkable push_heap(h, $xs) setup=(h=MutableBinaryMinHeap{Float64}())
+SUITE[["heap","mutable", "min", "pop"]] =
+    @benchmarkable pop_heap(h) setup=(h=MutableBinaryMinHeap{Float64}($xs))

benchmark/pprinthelper.jl

@@ -0,0 +1,22 @@
+# This file was copied from Transducers.jl
+# which is available under an MIT license (see LICENSE).
+using PkgBenchmark
+using Markdown
+
+function displayresult(result)
+    md = sprint(export_markdown, result)
+    md = replace(md, ":x:" => "❌")
+    md = replace(md, ":white_check_mark:" => "✅")
+    display(Markdown.parse(md))
+end
+
+function printnewsection(name)
+    println()
+    println()
+    println()
+    printstyled("▃" ^ displaysize(stdout)[2]; color=:blue)
+    println()
+    printstyled(name; bold=true)
+    println()
+    println()
+end

benchmark/pprintjudge.jl

@@ -0,0 +1,15 @@
+# This file was copied from Transducers.jl
+# which is available under an MIT license (see LICENSE).
+using PkgBenchmark
+include("pprinthelper.jl")
+group_target = PkgBenchmark.readresults(joinpath(@__DIR__, "result-target.json"))
+group_baseline = PkgBenchmark.readresults(joinpath(@__DIR__, "result-baseline.json"))
+judgement = judge(group_target, group_baseline)
+
+displayresult(judgement)
+
+printnewsection("Target result")
+displayresult(group_target)
+
+printnewsection("Baseline result")
+displayresult(group_baseline)

benchmark/runbenchmarks.jl

@@ -0,0 +1,12 @@
+# This file was copied from Transducers.jl
+# which is available under an MIT license (see LICENSE).
+using PkgBenchmark
+benchmarkpkg(
+    dirname(@__DIR__),
+    BenchmarkConfig(
+        env = Dict(
+            "JULIA_NUM_THREADS" => "1",
+        ),
+    ),
+    resultfile = joinpath(@__DIR__, "result.json"),
+)

benchmark/runjudge.jl

@@ -0,0 +1,25 @@
+# This file was copied from Transducers.jl
+# which is available under an MIT license (see LICENSE).
+using PkgBenchmark
+
+mkconfig(; kwargs...) =
+    BenchmarkConfig(
+        env = Dict(
+            "JULIA_NUM_THREADS" => "1",
+        );
+        kwargs...
+    )
+
+group_target = benchmarkpkg(
+    dirname(@__DIR__),
+    mkconfig(),
+    resultfile = joinpath(@__DIR__, "result-target.json"),
+)
+
+group_baseline = benchmarkpkg(
+    dirname(@__DIR__),
+    mkconfig(id = "baseline"),
+    resultfile = joinpath(@__DIR__, "result-baseline.json"),
+)
+
+judgement = judge(group_target, group_baseline)