deleted and moved non-used files, updated benchmarking scripts, ran some new benchmarks

Author: robertfeldt

Rakefile

@@ -267,33 +267,34 @@ function multitest_opt(problemDescriptions, method; NumRepetitions = 3)
 
             start_time = time()
             ftn, dur = fitness_for_opt(prob, numdims, popsize, ceil(Int, numfevals), method)
-            df[:ElapsedTime] = dur
-            df[:StartTime] = Libc.strftime("%Y-%m-%d %H:%M:%S", start_time)
-            df[:Fitness] = ftn
+            df[!, :ElapsedTime] = [dur]
+            df[!, :StartTime] = [Libc.strftime("%Y-%m-%d %H:%M:%S", start_time)]
+            df[!, :Fitness] = [ftn]
 
             push!(dfs, df)
         end
     end
 
-    vcat(dfs)
+    vcat(dfs...)
 end
 
 using CSV
 
 function read_benchmark_db(filename)
     if isfile(filename)
-        return CSV.read(filename)
+        return CSV.read(filename, DataFrame)
     else
         @warn "Benchmark results file $filename not found, returning empty frame"
         return DataFrame()
     end
 end
 
 function add_rank_per_group(df, groupcols, rankcol, resultcol)
-    by(df, groupcols) do subdf
-        orderedsubdf = subdf[sortperm(subdf[:,rankcol]), :]
-        orderedsubdf[resultcol] = collect(1:size(orderedsubdf,1))
-        return orderedsubdf
+    gdf = groupby(df, groupcols)
+    combine(gdf) do subdf
+        ordered = subdf[sortperm(subdf[:, rankcol]), :]
+        ordered[!, resultcol] = collect(1:size(ordered,1))
+        return ordered
     end
 end
 
@@ -303,45 +304,45 @@ function list_benchmark_db(db, saveResultCsvFile = nothing)
 
     if numrows > 1
         # Find min fitness per problem
-        minfitpp = by(db, [:Problem, :NumDims]) do df
-            DataFrame(
-                MinFitness = minimum(df[:Fitness])
-            )
+        minfitpp = combine(groupby(db, [:Problem, :NumDims])) do df
+            DataFrame(MinFitness = minimum(df.Fitness))
         end
 
         # Add col with order of magnitude worse than min fitness for each run
-        db = join(db, minfitpp, on = [:Problem, :NumDims])
-        db[:LogTimesWorseFitness] = log10.(db[:Fitness] ./ db[:MinFitness])
+        db = leftjoin(db, minfitpp, on = [:Problem, :NumDims])
+        db[:, :LogTimesWorseFitness] = log10.(db.Fitness ./ db.MinFitness)
 
         # Calc median fitness and time per problem and method.
-        sumdf = by(db, [:Problem, :NumDims, :Method]) do df
+        sumdf = combine(groupby(db, [:Problem, :NumDims, :Method])) do df
             DataFrame(N = size(df, 1),
-                      MedianFitness = median(df[:,:Fitness]),
-                      MedianTime = median(df[:,:ElapsedTime]))
+                      MedianFitness = median(df.Fitness),
+                      MedianTime = median(df.ElapsedTime))
         end
 
         # Rank on median fitness and median time for each problem.
         sumdf = add_rank_per_group(sumdf, [:Problem, :NumDims], :MedianFitness, :RankFitness)
         sumdf = add_rank_per_group(sumdf, [:Problem, :NumDims], :MedianTime, :RankTime)
 
         # Get number of runs and median magnitude worse per method
-        permethod = by(db, [:Method]) do df
+        permethod = combine(groupby(db, [:Method])) do df
             DataFrame(
                 NumRuns = size(df, 1),
-                MedianLogTimesWorseFitness = round(median(df[:, :LogTimesWorseFitness]), digits=1)
+                MedianLogTimesWorseFitness = round(median(df.LogTimesWorseFitness), digits=1)
             )
         end
 
         # and merge with table with mean ranks of fitness and time.
-        summarydf = by(sumdf, [:Method]) do df
+        summarydf = combine(groupby(sumdf, [:Method])) do df
+            rfs = df.RankFitness
+            rts = df.RankTime
             DataFrame(
-                MeanRank = round(mean(df[:RankFitness]), digits=3),
-                Num1sFitness = sum(r -> (r == 1) ? 1 : 0, df[:RankFitness]),
-                MeanRankTime = round(mean(df[:RankTime]), digits=3),
-                Num1sTime = sum(r -> (r == 1) ? 1 : 0, df[:RankTime]),
+                MeanRank = round(mean(rfs), digits=3),
+                Num1sFitness = sum(r -> (r == 1) ? 1 : 0, rfs),
+                MeanRankTime = round(mean(rts), digits=3),
+                Num1sTime = sum(r -> (r == 1) ? 1 : 0, rts),
             )
         end
-        df = join(summarydf, permethod, on = :Method)
+        df = leftjoin(summarydf, permethod, on = :Method)
 
         # Now sort and print
         sort!(df, [:MeanRank, :MeanRankTime, :Num1sFitness])
@@ -381,14 +382,14 @@ function compare_optimizers_to_benchmarks(benchmarkfile, pset, optimizers, nreps
         totalruns = length(optimizers) * nreps * length(pset)
         runnum = 0
         for optmethod in optimizers
-            optsel = db[:Method] .== string(optmethod)
+            optsel = db.Method .== string(optmethod)
             for pd in pset
                 probname, numdims, popsize, numfevals = pd
-                psel = db[:Problem] .== probname
-                dsel = db[:NumDims] .== numdims
+                psel = db.Problem .== probname
+                dsel = db.NumDims .== numdims
                 df = db[optsel .& psel .& dsel, :]
-                benchfitnesses = convert(Vector{Float64}, df[:Fitness])
-                benchtimes = convert(Vector{Float64}, df[:ElapsedTime])
+                benchfitnesses = convert(Vector{Float64}, df.Fitness)
+                benchtimes = convert(Vector{Float64}, df.ElapsedTime)
                 newfs = Float64[]
                 newtimes = Float64[]
                 prob = BlackBoxOptim.example_problems[probname]
@@ -437,19 +438,19 @@ function compare_optimizers_to_benchmarks(benchmarkfile, pset, optimizers, nreps
 
         # Use Benjamini-Hochberg to judge which pvalues are significant given we did
         # many comparisons.
-        ftn_pvs = convert(Vector{Float64}, df[:FitnessPvalue])
-        df[:FitnessSignificantBH001] = benjamini_hochberg(ftn_pvs, 0.01)
-        df[:FitnessSignificantBH005] = benjamini_hochberg(ftn_pvs, 0.05)
-        df[:FitnessSignificantBH010] = benjamini_hochberg(ftn_pvs, 0.10)
+        ftn_pvs = convert(Vector{Float64}, df.FitnessPvalue)
+        df[!, :FitnessSignificantBH001] = benjamini_hochberg(ftn_pvs, 0.01)
+        df[!, :FitnessSignificantBH005] = benjamini_hochberg(ftn_pvs, 0.05)
+        df[!, :FitnessSignificantBH010] = benjamini_hochberg(ftn_pvs, 0.10)
 
-        time_pvs = convert(Vector{Float64}, df[:TimePvalue])
-        df[:TimeSignificantBH001] = benjamini_hochberg(time_pvs, 0.01)
-        df[:TimeSignificantBH005] = benjamini_hochberg(time_pvs, 0.05)
-        df[:TimeSignificantBH010] = benjamini_hochberg(time_pvs, 0.10)
+        time_pvs = convert(Vector{Float64}, df.TimePvalue)
+        df[!, :TimeSignificantBH001] = benjamini_hochberg(time_pvs, 0.01)
+        df[!, :TimeSignificantBH005] = benjamini_hochberg(time_pvs, 0.05)
+        df[!, :TimeSignificantBH010] = benjamini_hochberg(time_pvs, 0.10)
 
         CSV.write(Libc.strftime("comparison_%Y%m%d_%H%M%S.csv", time()), df)
     else
-        df = CSV.read(comparisonfile)
+        df = CSV.read(comparisonfile, DataFrame)
     end
     sort!(df, [:FitnessPvalue])
     report_below_pvalue(df, col_prefix="Fitness", pvalue=1.00)
@@ -461,14 +462,14 @@ function compare_optimizers_to_benchmarks(benchmarkfile, pset, optimizers, nreps
 
     # Report (in color) on number of significant differences after Benjamini-Hochberg
     # correction.
-    n_ftn_reg = sum(df[:FitnessSignificantBH005] .& (df[:FitnessOrder] .== "<"))
-    n_ftn_imp = sum(df[:FitnessSignificantBH005] .& (df[:FitnessOrder] .== ">"))
+    n_ftn_reg = sum(df.FitnessSignificantBH005 .& (df.FitnessOrder .== "<"))
+    n_ftn_imp = sum(df.FitnessSignificantBH005 .& (df.FitnessOrder .== ">"))
     printstyled("\n$n_ftn_reg significant fitness regressions at Benjamini-Hochberg 0.05 level\n",
                 color=n_ftn_reg > 0 ? :red : :green)
     printstyled("\n$n_ftn_imp significant fitness improvments at Benjamini-Hochberg 0.05 level\n",
                 color=n_ftn_imp > 0 ? :green : :white)
-    n_time_reg = sum(df[:TimeSignificantBH005] .& (df[:TimeOrder] .== "<"))
-    n_time_imp = sum(df[:TimeSignificantBH005] .& (df[:TimeOrder] .== ">"))
+    n_time_reg = sum(df.TimeSignificantBH005 .& (df.TimeOrder .== "<"))
+    n_time_imp = sum(df.TimeSignificantBH005 .& (df.TimeOrder .== ">"))
     printstyled("\n$n_time_reg significant time regressions at Benjamini-Hochberg 0.05 level\n",
                 color=n_time_reg > 0 ? :red : :green)
     printstyled("\n$n_time_imp significant time improvments at Benjamini-Hochberg 0.05 level\n",
@@ -491,7 +492,7 @@ function benjamini_hochberg(pvals, alpha = 0.05)
 end
 
 function report_below_pvalue(df; col_prefix="Fitness", pvalue = 0.05)
-    selection = df[Symbol(col_prefix, "Pvalue")] .< pvalue
+    selection = df[:, Symbol(col_prefix, "Pvalue")] .< pvalue
     log("Num problems with $col_prefix p-values < $(pvalue): $(sum(selection))\n")
     # workaround sum(isequal, []) throws
     num_new_worse = sum(df[selection, Symbol(col_prefix, "Order")] .== "<")
@@ -502,4 +503,4 @@ function report_below_pvalue(df; col_prefix="Fitness", pvalue = 0.05)
     any(selection) && println(df[selection, :])
 end
 
-@CPUtime main(ARGS)
+@CPUtime main(ARGS)

TODO.md design/TODO.md

@@ -1,16 +1,16 @@
-"Method","MeanRank","Num1sFitness","MeanRankTime","Num1sTime","NumRuns","MedianLogTimesWorseFitness"
-"dxnes",3.87,5,8.087,2,230,0.3
-"adaptive_de_rand_1_bin_radiuslimited",4.913,1,9.304,0,230,0.9
-"xnes",5.087,4,13.13,0,230,2.2
-"de_rand_1_bin_radiuslimited",5.174,3,7.304,1,230,1.0
-"adaptive_de_rand_1_bin",5.478,1,9.087,0,230,1.7
-"generating_set_search",5.783,3,2.739,10,230,0.5
-"de_rand_1_bin",6.696,1,7.652,0,230,3.2
-"separable_nes",8.217,1,4.783,0,230,8.5
-"resampling_inheritance_memetic_search",8.304,2,5.739,1,230,8.4
-"probabilistic_descent",8.565,0,5.609,7,230,6.3
-"resampling_memetic_search",8.957,1,5.13,2,230,8.3
-"de_rand_2_bin_radiuslimited",8.957,1,11.217,0,230,5.7
-"de_rand_2_bin",11.565,0,9.696,0,230,8.8
-"random_search",14.13,0,13.435,0,230,11.7
-"simultaneous_perturbation_stochastic_approximation",14.304,0,7.087,0,230,11.6
+Method,MeanRank,Num1sFitness,MeanRankTime,Num1sTime,NumRuns,MedianLogTimesWorseFitness
+adaptive_de_rand_1_bin,4.609,3,9.261,0,46,1.6
+de_rand_1_bin_radiuslimited,4.826,2,6.478,0,46,0.5
+adaptive_de_rand_1_bin_radiuslimited,4.826,1,8.783,0,46,0.6
+xnes,5.261,6,14.565,0,46,1.8
+generating_set_search,5.565,5,4.304,8,46,0.3
+dxnes,6.13,0,14.304,0,46,0.5
+de_rand_1_bin,6.739,1,8.826,0,46,2.9
+separable_nes,8.087,2,4.261,1,46,8.6
+resampling_inheritance_memetic_search,8.261,1,4.522,6,46,7.9
+de_rand_2_bin_radiuslimited,8.522,0,9.174,0,46,5.3
+probabilistic_descent,8.826,0,5.739,5,46,7.2
+resampling_memetic_search,8.913,2,5.913,2,46,8.4
+de_rand_2_bin,11.13,0,9.696,0,46,8.9
+random_search,14.087,0,5.304,1,46,11.2
+simultaneous_perturbation_stochastic_approximation,14.217,0,8.87,0,46,11.8

examples/benchmarking/benchmark_runs_210516.csv

@@ -0,0 +1,2 @@
+Circa 2015-2021: Apple MacBook Pro 13" 2015, different Julia versions
+20210516: Apple MacBook Air with Apple M1 processor, Julia 1.6.1

examples/benchmarking/compare_optimizers.jl

@@ -10,6 +10,6 @@ end
 
 nreps = (length(ARGS) > 0 ? parse(Int, ARGS[2]) : 10)
 
-cmd = `$(Sys.BINDIR)/julia --color=yes -L $scripts_dir/activate_pwd_env.jl -- $scripts_dir/compare_optimizers.jl update --benchmarkfile $nbfile --numreps $nreps`
+cmd = `$(Sys.BINDIR)/julia --color=yes -L $scripts_dir/activate_pwd_env.jl -- $scripts_dir/compare_optimizers.jl update --benchmarkfile $nbfile --numreps $nreps --optimizers all`
 
 run(cmd)

examples/benchmarking/latest_toplist.csv

@@ -1 +1 @@
-julia --color=yes -L ../../src/BlackBoxOptim.jl compare_optimizers.jl list -o latest_toplist.csv --benchmarkfile benchmark_runs_150913.csv
+julia --color=yes compare_optimizers.jl list -o latest_toplist.csv --benchmarkfile benchmark_runs_210516.csv