Skip to content

Commit eee2305

Browse files
anatoliykmetyukanatoliykmetyuk
authored
Merge pull request #13787 from dotty-staging/mb/bench-jmh-options
Allow passing any JMH option to scala3-bench
2 parents 9f06976 + 30d600c commit eee2305

16 files changed

+89
-75
lines changed

‎bench/profiles/ci.yml

+15-15
Original file line numberDiff line numberDiff line change
@@ -1,55 +1,55 @@
11
scripts:
22

33
re2s:
4-
- measure 1 1 1 $(find $PROG_HOME/tests/re2s/src -name *.scala)
4+
- measure -wi 1 -i 1 -f 1 -- $(find $PROG_HOME/tests/re2s/src -name *.scala)
55

66
implicit-cache:
7-
- measure 1 1 1 $PROG_HOME/dotty/tests/bench/implicit_cache.scala
7+
- measure -wi 1 -i 1 -f 1 -- $PROG_HOME/dotty/tests/bench/implicit_cache.scala
88

99
implicit-cache-from-tasty:
1010
- source $PROG_HOME/dotty/bench/scripts/implicit-cache-from-tasty
1111

1212
implicit-scope-loop:
13-
- measure 1 1 1 $PROG_HOME/dotty/tests/bench/implicit-scope-loop.scala
13+
- measure -wi 1 -i 1 -f 1 -- $PROG_HOME/dotty/tests/bench/implicit-scope-loop.scala
1414

1515
i1535:
16-
- measure 1 1 1 $PROG_HOME/dotty/tests/bench/i1535.scala
16+
- measure -wi 1 -i 1 -f 1 -- $PROG_HOME/dotty/tests/bench/i1535.scala
1717

1818
i1687:
19-
- measure 1 1 1 $PROG_HOME/dotty/tests/bench/i1687.scala
19+
- measure -wi 1 -i 1 -f 1 -- $PROG_HOME/dotty/tests/bench/i1687.scala
2020

2121
empty-class:
22-
- measure 1 1 1 $PROG_HOME/dotty/tests/bench/empty-class.scala
22+
- measure -wi 1 -i 1 -f 1 -- $PROG_HOME/dotty/tests/bench/empty-class.scala
2323

2424
empty-object:
25-
- measure 1 1 1 $PROG_HOME/dotty/tests/bench/empty-object.scala
25+
- measure -wi 1 -i 1 -f 1 -- $PROG_HOME/dotty/tests/bench/empty-object.scala
2626

2727
empty-file:
28-
- measure 1 1 1 $PROG_HOME/dotty/tests/bench/empty-file.scala
28+
- measure -wi 1 -i 1 -f 1 -- $PROG_HOME/dotty/tests/bench/empty-file.scala
2929

3030
patmatexhaust:
31-
- measure 1 1 1 $PROG_HOME/dotty/tests/bench/patmatexhaust.scala
31+
- measure -wi 1 -i 1 -f 1 -- $PROG_HOME/dotty/tests/bench/patmatexhaust.scala
3232

3333
exhaustivity-I:
34-
- measure 1 1 1 $PROG_HOME/dotty/tests/bench/exhaustivity-I.scala
34+
- measure -wi 1 -i 1 -f 1 -- $PROG_HOME/dotty/tests/bench/exhaustivity-I.scala
3535

3636
exhaustivity-S:
37-
- measure 1 1 1 $PROG_HOME/dotty/tests/bench/exhaustivity-S.scala
37+
- measure -wi 1 -i 1 -f 1 -- $PROG_HOME/dotty/tests/bench/exhaustivity-S.scala
3838

3939
exhaustivity-T:
40-
- measure 1 1 1 $PROG_HOME/dotty/tests/bench/exhaustivity-T.scala
40+
- measure -wi 1 -i 1 -f 1 -- $PROG_HOME/dotty/tests/bench/exhaustivity-T.scala
4141

4242
exhaustivity-V:
43-
- measure 1 1 1 $PROG_HOME/dotty/tests/bench/exhaustivity-V.scala
43+
- measure -wi 1 -i 1 -f 1 -- $PROG_HOME/dotty/tests/bench/exhaustivity-V.scala
4444

4545
implicitNums:
46-
- measure 1 1 1 $PROG_HOME/dotty/tests/bench/implicitNums.scala
46+
- measure -wi 1 -i 1 -f 1 -- $PROG_HOME/dotty/tests/bench/implicitNums.scala
4747

4848
implicitNums-from-tasty:
4949
- source $PROG_HOME/dotty/bench/scripts/implicitNums-from-tasty
5050

5151
inductive-implicits:
52-
- measure 1 1 1 $PROG_HOME/dotty/tests/bench/inductive-implicits.scala
52+
- measure -wi 1 -i 1 -f 1 -- $PROG_HOME/dotty/tests/bench/inductive-implicits.scala
5353

5454
scalap:
5555
- source $PROG_HOME/dotty/bench/scripts/scalap

‎bench/profiles/empty.yml

+3-3
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,13 @@ charts:
2121
scripts:
2222

2323
empty-class:
24-
- measure $PROG_HOME/dotty/tests/bench/empty-class.scala
24+
- measure -- $PROG_HOME/dotty/tests/bench/empty-class.scala
2525

2626
empty-object:
27-
- measure $PROG_HOME/dotty/tests/bench/empty-object.scala
27+
- measure -- $PROG_HOME/dotty/tests/bench/empty-object.scala
2828

2929
empty-file:
30-
- measure $PROG_HOME/dotty/tests/bench/empty-file.scala
30+
- measure -- $PROG_HOME/dotty/tests/bench/empty-file.scala
3131

3232
config:
3333
pr_base_url: "https://github.com/lampepfl/dotty/pull/"

‎bench/profiles/exhaustivity.yml

+8-8
Original file line numberDiff line numberDiff line change
@@ -50,28 +50,28 @@ charts:
5050
scripts:
5151

5252
patmatexhaust:
53-
- measure 20 40 3 $PROG_HOME/dotty/tests/bench/patmatexhaust.scala
53+
- measure -wi 20 -i 40 -f 3 -- $PROG_HOME/dotty/tests/bench/patmatexhaust.scala
5454

5555
exhaustivity-I:
56-
- measure 20 40 3 $PROG_HOME/dotty/tests/bench/exhaustivity-I.scala
56+
- measure -wi 20 -i 40 -f 3 -- $PROG_HOME/dotty/tests/bench/exhaustivity-I.scala
5757

5858
exhaustivity-S:
59-
- measure 20 40 3 $PROG_HOME/dotty/tests/bench/exhaustivity-S.scala
59+
- measure -wi 20 -i 40 -f 3 -- $PROG_HOME/dotty/tests/bench/exhaustivity-S.scala
6060

6161
exhaustivity-T:
62-
- measure 20 40 3 $PROG_HOME/dotty/tests/bench/exhaustivity-T.scala
62+
- measure -wi 20 -i 40 -f 3 -- $PROG_HOME/dotty/tests/bench/exhaustivity-T.scala
6363

6464
exhaustivity-V:
65-
- measure 20 40 3 $PROG_HOME/dotty/tests/bench/exhaustivity-V.scala
65+
- measure -wi 20 -i 40 -f 3 -- $PROG_HOME/dotty/tests/bench/exhaustivity-V.scala
6666

6767
exhaustivity-mips:
68-
- measure 20 40 3 $PROG_HOME/dotty/tests/patmat/i7186.scala
68+
- measure -wi 20 -i 40 -f 3 -- $PROG_HOME/dotty/tests/patmat/i7186.scala
6969

7070
exhaustivity-i12241:
71-
- measure 20 40 3 $PROG_HOME/dotty/tests/patmat/i12241.scala
71+
- measure -wi 20 -i 40 -f 3 -- $PROG_HOME/dotty/tests/patmat/i12241.scala
7272

7373
exhaustivity-i12358:
74-
- measure 20 40 3 $PROG_HOME/dotty/tests/patmat/i12358.scala
74+
- measure -wi 20 -i 40 -f 3 -- $PROG_HOME/dotty/tests/patmat/i12358.scala
7575

7676
config:
7777
pr_base_url: "https://github.com/lampepfl/dotty/pull/"

‎bench/profiles/implicits.yml

+4-4
Original file line numberDiff line numberDiff line change
@@ -30,19 +30,19 @@ charts:
3030
scripts:
3131

3232
implicit-cache:
33-
- measure $PROG_HOME/dotty/tests/bench/implicit_cache.scala
33+
- measure -- $PROG_HOME/dotty/tests/bench/implicit_cache.scala
3434

3535
implicit-cache-from-tasty:
3636
- source $PROG_HOME/dotty/bench/scripts/implicit-cache-from-tasty
3737

3838
implicit-scope-loop:
39-
- measure $PROG_HOME/dotty/tests/bench/implicit-scope-loop.scala
39+
- measure -- $PROG_HOME/dotty/tests/bench/implicit-scope-loop.scala
4040

4141
inductive-implicits:
42-
- measure $PROG_HOME/dotty/tests/bench/inductive-implicits.scala
42+
- measure -- $PROG_HOME/dotty/tests/bench/inductive-implicits.scala
4343

4444
implicitNums:
45-
- measure $PROG_HOME/dotty/tests/bench/implicitNums.scala
45+
- measure -- $PROG_HOME/dotty/tests/bench/implicitNums.scala
4646

4747
implicitNums-from-tasty:
4848
- source $PROG_HOME/dotty/bench/scripts/implicitNums-from-tasty

‎bench/profiles/misc.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ charts:
1515
scripts:
1616

1717
i1535:
18-
- measure $PROG_HOME/dotty/tests/bench/i1535.scala
18+
- measure -- $PROG_HOME/dotty/tests/bench/i1535.scala
1919

2020
i1687:
21-
- measure $PROG_HOME/dotty/tests/bench/i1687.scala
21+
- measure -- $PROG_HOME/dotty/tests/bench/i1687.scala

‎bench/profiles/projects.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -26,10 +26,10 @@ charts:
2626

2727
scripts:
2828
dotty:
29-
- measure -with-compiler $(find $PROG_HOME/dotty/compiler/src/dotty -name *.scala -o -name *.java)
29+
- measure -- -with-compiler $ $(find $PROG_HOME/dotty/compiler/src/dotty -name *.scala -o -name *.java)
3030

3131
re2s:
32-
- measure $(find $PROG_HOME/tests/re2s/src -name *.scala)
32+
- measure -- $(find $PROG_HOME/tests/re2s/src -name *.scala)
3333

3434
# scalapb:
3535
# - source $PROG_HOME/dotty/bench/scripts/scalapb

‎bench/profiles/sbt.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ charts:
99

1010
scripts:
1111
dotty-sbt:
12-
- measure -with-compiler -Yforce-sbt-phases -with-dotty $(find $PROG_HOME/dotty/compiler/src/dotty -name *.scala -o -name *.java)
12+
- measure -- -with-compiler $ -Yforce-sbt-phases -with-dotty $(find $PROG_HOME/dotty/compiler/src/dotty -name *.scala -o -name *.java)
1313

1414
config:
1515
pr_base_url: "https://github.com/lampepfl/dotty/pull/"

‎bench/profiles/tuples.yml

+5-5
Original file line numberDiff line numberDiff line change
@@ -44,19 +44,19 @@ charts:
4444
scripts:
4545

4646
tuple22-creation-apply:
47-
- measure $PROG_HOME/dotty/tests/bench/tuple22-creation-apply.scala
47+
- measure -- $PROG_HOME/dotty/tests/bench/tuple22-creation-apply.scala
4848

4949
tuple22-creation-cons:
50-
- measure $PROG_HOME/dotty/tests/bench/tuple22-creation-cons.scala
50+
- measure -- $PROG_HOME/dotty/tests/bench/tuple22-creation-cons.scala
5151

5252
tuple22-tails:
53-
- measure $PROG_HOME/dotty/tests/bench/tuple22-tails.scala
53+
- measure -- $PROG_HOME/dotty/tests/bench/tuple22-tails.scala
5454

5555
tuple22-apply:
56-
- measure $PROG_HOME/dotty/tests/bench/tuple22-apply.scala
56+
- measure -- $PROG_HOME/dotty/tests/bench/tuple22-apply.scala
5757

5858
tuple22-size:
59-
- measure $PROG_HOME/dotty/tests/bench/tuple22-size.scala
59+
- measure -- $PROG_HOME/dotty/tests/bench/tuple22-size.scala
6060

6161
tuple-reverse:
6262
- measure-run TupleOps.reverse

‎bench/profiles/typing.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ charts:
88
scripts:
99

1010
find-ref:
11-
- measure $PROG_HOME/dotty/tests/bench/FindRef.scala
11+
- measure -- $PROG_HOME/dotty/tests/bench/FindRef.scala
1212

1313
config:
1414
pr_base_url: "https://github.com/lampepfl/dotty/pull/"

‎bench/scripts/collection-vector.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
11
#!/usr/bin/env bash
2-
sbt "dotty-bench-bootstrapped/jmh:run 40 40 3 bench/tests/Vector.scala"
2+
sbt "dotty-bench-bootstrapped/jmh:run -wi 40 -i 40 -f 3 -- bench/tests/Vector.scala"

‎bench/scripts/compiler-cold.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
11
#!/usr/bin/env bash
2-
find compiler/src/ -type f \( -name "*.scala" -or -name "*.java" \) -exec echo "dotty-bench-bootstrapped/jmh:run 0 1 10" {} + | sbt
2+
find compiler/src/ -type f \( -name "*.scala" -or -name "*.java" \) -exec echo "dotty-bench-bootstrapped/jmh:run -wi 0 -i 1 -f 10 -- " {} + | sbt

‎bench/scripts/compiler.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
11
#!/usr/bin/env bash
2-
find compiler/src/ -type f \( -name "*.scala" -or -name "*.java" \) -exec echo "dotty-bench-bootstrapped/jmh:run 5 10" {} + | sbt
2+
find compiler/src/ -type f \( -name "*.scala" -or -name "*.java" \) -exec echo "dotty-bench-bootstrapped/jmh:run -wi 5 -i 10 -- " {} + | sbt

‎bench/scripts/library-cold.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
11
#!/usr/bin/env bash
2-
find library/src/ -type f \( -name "*.scala" -or -name "*.java" \) -exec echo "dotty-bench-bootstrapped/jmh:run 0 1 10" {} + | sbt
2+
find library/src/ -type f \( -name "*.scala" -or -name "*.java" \) -exec echo "dotty-bench-bootstrapped/jmh:run -wi 0 -i 1 -f 10 -- " {} + | sbt

‎bench/scripts/library.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
11
#!/usr/bin/env bash
2-
find library/src/ -type f \( -name "*.scala" -or -name "*.java" \) -exec echo "dotty-bench-bootstrapped/jmh:run 40 30" {} + | sbt
2+
find library/src/ -type f \( -name "*.scala" -or -name "*.java" \) -exec echo "dotty-bench-bootstrapped/jmh:run -wi 40 -i 30 -- " {} + | sbt

‎bench/src/main/scala/Benchmarks.scala

+40-26
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ import reporting._
77

88
import org.openjdk.jmh.results.RunResult
99
import org.openjdk.jmh.runner.Runner
10-
import org.openjdk.jmh.runner.options.OptionsBuilder
10+
import org.openjdk.jmh.runner.options.{OptionsBuilder, CommandLineOptions}
1111
import org.openjdk.jmh.annotations._
1212
import org.openjdk.jmh.results.format._
1313
import java.util.concurrent.TimeUnit
@@ -22,37 +22,47 @@ import dotty.tools.io.AbstractFile
2222
object Bench {
2323
val COMPILE_OPTS_FILE = "compile.txt"
2424

25+
def printUsage() =
26+
println("Usage (from SBT): scala3-bench/jmh:run <JMH arguments> -- <scalac arguments>")
27+
println("Display JMH help: scala3-bench/jmh:run -h")
28+
println("Our default JMH options: -wi 30 -i 20 -f 3 -tu ms -bm AverageTime -jvmArgs \"-Xms2G -Xmx2G\"")
29+
2530
def main(args: Array[String]): Unit = {
2631
if (args.isEmpty) {
27-
println("Missing <args>")
32+
println("Missing arguments.")
33+
printUsage()
2834
return
2935
}
30-
val (intArgs, args1) = args.span(x => try { x.toInt; true } catch { case _: Throwable => false } )
3136

32-
val warmup = if (intArgs.length > 0) intArgs(0).toInt else 30
33-
val iterations = if (intArgs.length > 1) intArgs(1).toInt else 20
34-
val forks = if (intArgs.length > 2) intArgs(2).toInt else 1
37+
val (jmhArgs, _scalacArgs) = args.span(_ != "--")
38+
val scalacArgs = _scalacArgs.drop(1)
3539

40+
storeCompileOptions(scalacArgs)
3641

37-
import File.{ separator => sep }
42+
val jmhCliOps = new CommandLineOptions(jmhArgs:_*)
43+
val jmhOps = new OptionsBuilder().parent(jmhCliOps)
44+
45+
// set our own default options
46+
if !jmhCliOps.shouldFailOnError().hasValue() then jmhOps.shouldFailOnError(true)
47+
if !jmhCliOps.getWarmupIterations().hasValue() then jmhOps.warmupIterations(30)
48+
if !jmhCliOps.getMeasurementIterations().hasValue() then jmhOps.measurementIterations(20)
49+
if !jmhCliOps.getForkCount().hasValue() then jmhOps.forks(1)
50+
if jmhCliOps.getBenchModes().isEmpty() then jmhOps.mode(Mode.AverageTime)
51+
if !jmhCliOps.getTimeUnit().hasValue() then jmhOps.timeUnit(TimeUnit.MILLISECONDS)
52+
if !jmhCliOps.getJvmArgs().hasValue() then jmhOps.jvmArgs("-Xms2G", "-Xmx2G")
53+
54+
val runner = new Runner(jmhOps.build())
55+
56+
if jmhCliOps.shouldHelp() then
57+
printUsage()
58+
println("Following is the JMH options documentation.")
59+
println("-------------------------------------------")
60+
return jmhCliOps.showHelp()
61+
if jmhCliOps.shouldList() then return runner.list()
62+
if jmhCliOps.shouldListWithParams() then return runner.listWithParams(jmhCliOps)
63+
if jmhCliOps.shouldListProfilers() then return jmhCliOps.listProfilers()
64+
if jmhCliOps.shouldListResultFormats() then return jmhCliOps.listResultFormats()
3865

39-
val args2 = args1.map { arg =>
40-
if ((arg.endsWith(".scala") || arg.endsWith(".java")) && !(new File(arg)).isAbsolute) ".." + sep + arg
41-
else arg
42-
}
43-
storeCompileOptions(args2)
44-
45-
val opts = new OptionsBuilder()
46-
.shouldFailOnError(true)
47-
.jvmArgs("-Xms2G", "-Xmx2G")
48-
.mode(Mode.AverageTime)
49-
.timeUnit(TimeUnit.MILLISECONDS)
50-
.warmupIterations(warmup)
51-
.measurementIterations(iterations)
52-
.forks(forks)
53-
.build
54-
55-
val runner = new Runner(opts) // full access to all JMH features, you can also provide a custom output Format here
5666
runner.run() // actually run the benchmarks
5767

5868
removeCompileOptions
@@ -61,13 +71,17 @@ object Bench {
6171
def removeCompileOptions: Unit = new File(COMPILE_OPTS_FILE).delete()
6272

6373
def storeCompileOptions(args: Array[String]): Unit = {
74+
import File.{ separator => sep }
75+
6476
val standard_libs = System.getProperty("BENCH_CLASS_PATH")
6577
val compiler_libs = System.getProperty("BENCH_COMPILER_CLASS_PATH")
6678

6779
val libs = if (args.contains("-with-compiler")) compiler_libs else standard_libs
68-
var argsNorm = args.filter(_ != "-with-compiler")
80+
var argsNorm = args.filter(_ != "-with-compiler").map { arg =>
81+
if ((arg.endsWith(".scala") || arg.endsWith(".java")) && !(new File(arg)).isAbsolute) ".." + sep + arg
82+
else arg
83+
}
6984

70-
import File.{ pathSeparator => sep }
7185
var cpIndex = argsNorm.indexOf("-classpath")
7286
if (cpIndex == -1) cpIndex = argsNorm.indexOf("-cp")
7387
if (cpIndex != -1) argsNorm(cpIndex + 1) = argsNorm(cpIndex + 1) + sep + libs

‎project/scripts/bootstrapCmdTests

+3-3
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,10 @@ set -e
55
source $(dirname $0)/cmdTestsCommon.inc.sh
66

77
# check that benchmarks can run
8-
"$SBT" "scala3-bench/jmh:run 1 1 tests/pos/alias.scala"
8+
"$SBT" "scala3-bench/jmh:run -wi 1 -i 1 -- tests/pos/alias.scala"
99
# The above is here as it relies on the bootstrapped library.
10-
"$SBT" "scala3-bench-bootstrapped/jmh:run 1 1 tests/pos/alias.scala"
11-
"$SBT" "scala3-bench-bootstrapped/jmh:run 1 1 -with-compiler compiler/src/dotty/tools/dotc/core/Types.scala"
10+
"$SBT" "scala3-bench-bootstrapped/jmh:run -wi 1 -i 1 -- tests/pos/alias.scala"
11+
"$SBT" "scala3-bench-bootstrapped/jmh:run -wi 1 -i 1 -- -with-compiler compiler/src/dotty/tools/dotc/core/Types.scala"
1212

1313
echo "testing scala.quoted.Expr.run from sbt scala"
1414
"$SBT" ";scala3-compiler-bootstrapped/scalac -with-compiler tests/run-staging/quote-run.scala; scala3-compiler-bootstrapped/scala -with-compiler Test" > "$tmp"

0 commit comments

Comments
 (0)