From f5e759ac3b89356353a84eb59299a45eb63a8738 Mon Sep 17 00:00:00 2001 From: Miles Frain Date: Tue, 24 Sep 2019 21:13:20 -0700 Subject: [PATCH 1/2] PkgBenchmark to detect performance regressions --- .gitignore | 1 + .travis.yml | 17 ++++++++++++++++ benchmark/Project.toml | 3 +++ benchmark/benchmarks.jl | 41 ++++++++++++++++++++++++++++++++++++++ benchmark/pprinthelper.jl | 22 ++++++++++++++++++++ benchmark/pprintjudge.jl | 15 ++++++++++++++ benchmark/runbenchmarks.jl | 12 +++++++++++ benchmark/runjudge.jl | 25 +++++++++++++++++++++++ 8 files changed, 136 insertions(+) create mode 100644 benchmark/Project.toml create mode 100644 benchmark/benchmarks.jl create mode 100644 benchmark/pprinthelper.jl create mode 100644 benchmark/pprintjudge.jl create mode 100644 benchmark/runbenchmarks.jl create mode 100644 benchmark/runjudge.jl diff --git a/.gitignore b/.gitignore index ead5283f4..fb1a162b6 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ docs/build/ docs/site/ docs/Manifest.toml Manifest.toml +benchmark/*.json diff --git a/.travis.yml b/.travis.yml index 0ad1817e4..da200774a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,6 +22,23 @@ after_success: # https://juliadocs.github.io/Documenter.jl/stable/man/hosting/#.travis.yml-Configuration-1 jobs: include: + - name: "Benchmark" + julia: 1.2 + os: linux + before_script: + - git fetch origin '+refs/heads/master:refs/remotes/origin/master' + - git branch baseline origin/master + # Run benchmark outside `script` so that it's hidden by default: + - julia --project=benchmark -e ' + using Pkg; Pkg.instantiate(); + include("benchmark/runjudge.jl");' + + script: + - julia --project=benchmark -e ' + using Pkg; Pkg.instantiate(); + include("benchmark/pprintjudge.jl");' + after_success: skip + if: type = pull_request - stage: "Documentation" julia: 1.2 os: linux diff --git a/benchmark/Project.toml b/benchmark/Project.toml new file mode 100644 index 000000000..16acc4979 --- /dev/null +++ b/benchmark/Project.toml @@ -0,0 +1,3 @@ +[deps] +BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" +PkgBenchmark = "32113eaa-f34f-5b0d-bd6c-c81e245fc73d" diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl new file mode 100644 index 000000000..5ba66f37a --- /dev/null +++ b/benchmark/benchmarks.jl @@ -0,0 +1,41 @@ +using Pkg +tempdir = mktempdir() +Pkg.activate(tempdir) +Pkg.develop(PackageSpec(path=joinpath(@__DIR__, ".."))) +Pkg.add(["BenchmarkTools", "PkgBenchmark", "Random"]) +Pkg.resolve() + +using DataStructures +using BenchmarkTools +using Random + +function push_heap(h::AbstractHeap, xs::Vector{Float64}) + n = length(xs) + + for i = 1 : n + push!(h, xs[i]) + end +end + +function pop_heap(h::AbstractHeap) + n = length(h) + + for i = 1 : n + pop!(h) + end +end + +Random.seed!(0) +xs = rand(10^6) + +const SUITE = BenchmarkGroup() + +SUITE["heap"] = BenchmarkGroup(["binaryheap"]) +SUITE[["heap","basic", "min", "push"]] = + @benchmarkable push_heap(h, $xs) setup=(h=BinaryMinHeap{Float64}()) +SUITE[["heap","basic", "min", "pop"]] = + @benchmarkable pop_heap(h) setup=(h=BinaryMinHeap{Float64}($xs)) +SUITE[["heap","mutable", "min", "push"]] = + @benchmarkable push_heap(h, $xs) setup=(h=MutableBinaryMinHeap{Float64}()) +SUITE[["heap","mutable", "min", "pop"]] = + @benchmarkable pop_heap(h) setup=(h=MutableBinaryMinHeap{Float64}($xs)) diff --git a/benchmark/pprinthelper.jl b/benchmark/pprinthelper.jl new file mode 100644 index 000000000..bf31d851f --- /dev/null +++ b/benchmark/pprinthelper.jl @@ -0,0 +1,22 @@ +# This file was copied from Transducers.jl +# which is available under an MIT license (see LICENSE). +using PkgBenchmark +using Markdown + +function displayresult(result) + md = sprint(export_markdown, result) + md = replace(md, ":x:" => "❌") + md = replace(md, ":white_check_mark:" => "✅") + display(Markdown.parse(md)) +end + +function printnewsection(name) + println() + println() + println() + printstyled("▃" ^ displaysize(stdout)[2]; color=:blue) + println() + printstyled(name; bold=true) + println() + println() +end diff --git a/benchmark/pprintjudge.jl b/benchmark/pprintjudge.jl new file mode 100644 index 000000000..fb634e5e8 --- /dev/null +++ b/benchmark/pprintjudge.jl @@ -0,0 +1,15 @@ +# This file was copied from Transducers.jl +# which is available under an MIT license (see LICENSE). +using PkgBenchmark +include("pprinthelper.jl") +group_target = PkgBenchmark.readresults(joinpath(@__DIR__, "result-target.json")) +group_baseline = PkgBenchmark.readresults(joinpath(@__DIR__, "result-baseline.json")) +judgement = judge(group_target, group_baseline) + +displayresult(judgement) + +printnewsection("Target result") +displayresult(group_target) + +printnewsection("Baseline result") +displayresult(group_baseline) diff --git a/benchmark/runbenchmarks.jl b/benchmark/runbenchmarks.jl new file mode 100644 index 000000000..1f700707f --- /dev/null +++ b/benchmark/runbenchmarks.jl @@ -0,0 +1,12 @@ +# This file was copied from Transducers.jl +# which is available under an MIT license (see LICENSE). +using PkgBenchmark +benchmarkpkg( + dirname(@__DIR__), + BenchmarkConfig( + env = Dict( + "JULIA_NUM_THREADS" => "1", + ), + ), + resultfile = joinpath(@__DIR__, "result.json"), +) diff --git a/benchmark/runjudge.jl b/benchmark/runjudge.jl new file mode 100644 index 000000000..c0b560322 --- /dev/null +++ b/benchmark/runjudge.jl @@ -0,0 +1,25 @@ +# This file was copied from Transducers.jl +# which is available under an MIT license (see LICENSE). +using PkgBenchmark + +mkconfig(; kwargs...) = + BenchmarkConfig( + env = Dict( + "JULIA_NUM_THREADS" => "1", + ); + kwargs... + ) + +group_target = benchmarkpkg( + dirname(@__DIR__), + mkconfig(), + resultfile = joinpath(@__DIR__, "result-target.json"), +) + +group_baseline = benchmarkpkg( + dirname(@__DIR__), + mkconfig(id = "baseline"), + resultfile = joinpath(@__DIR__, "result-baseline.json"), +) + +judgement = judge(group_target, group_baseline) From a86b87303b336926a888a6e7b16f95e5995e9765 Mon Sep 17 00:00:00 2001 From: Miles Frain Date: Tue, 24 Sep 2019 21:28:00 -0700 Subject: [PATCH 2/2] notes on PkgBenchmark usage --- benchmark/README.md | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 benchmark/README.md diff --git a/benchmark/README.md b/benchmark/README.md new file mode 100644 index 000000000..0aea6d59c --- /dev/null +++ b/benchmark/README.md @@ -0,0 +1,30 @@ +## Notes on regression testing with PkgBenchmark + +### To troubleshoot benchmark script locally: +``` +julia --project=benchmark -e ' + using Pkg; Pkg.instantiate(); + include("benchmark/runbenchmarks.jl");' +``` + +### To compare against baseline locally: + +Note, must have a `baseline` branch, which will be the refrence point against the currently active branch. A common use case is to point the baseline to the previous commit. + +This can be accomplished with +``` +git branch baseline HEAD~ +``` + +If there are errors preventing branch creation (likely due to earlier local benchmarking), may force a repoint with +``` +git branch -f baseline HEAD~ +``` + +Then run this command: +``` +julia --project=benchmark -e ' + using Pkg; Pkg.instantiate(); + include("benchmark/runjudge.jl"); + include("benchmark/pprintjudge.jl");' +``` \ No newline at end of file