Add results diff

Author: jstarry

.github/workflows/benchmark.yml

@@ -42,3 +42,13 @@ jobs:
 
       - name: Results
         run: npm run results
+
+      - name: Build comment message
+        run: MSG=`(cd results_diff && cargo run)`
+
+      - name: Comment PR
+        uses: unsplash/comment-on-pr@master
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          msg: "$MSG"

results_diff/Cargo.toml

@@ -0,0 +1,9 @@
+[package]
+name = "results_diff"
+version = "0.1.0"
+authors = ["Justin Starry <[email protected]>"]
+edition = "2018"
+
+[dependencies]
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"

results_diff/results.json

@@ -0,0 +1 @@
+[{"framework":"wasm-bindgen-v0.2.47-keyed","benchmark":"01_run1k","values":[114.791]},{"framework":"wasm-bindgen-v0.2.47-keyed","benchmark":"02_replace1k","values":[127.208]},{"framework":"wasm-bindgen-v0.2.47-keyed","benchmark":"03_update10th1k_x16","values":[170.312]},{"framework":"wasm-bindgen-v0.2.47-keyed","benchmark":"04_select1k","values":[21.985]},{"framework":"wasm-bindgen-v0.2.47-keyed","benchmark":"05_swap1k","values":[58.483]},{"framework":"wasm-bindgen-v0.2.47-keyed","benchmark":"06_remove-one-1k","values":[23.557]},{"framework":"wasm-bindgen-v0.2.47-keyed","benchmark":"07_create10k","values":[1222.112,1105.367,1159.221,1166.379,1145.693]},{"framework":"wasm-bindgen-v0.2.47-keyed","benchmark":"08_create1k-after1k_x2","values":[238.403,222.106,237.156,294.204,248.482]},{"framework":"wasm-bindgen-v0.2.47-keyed","benchmark":"09_clear1k_x8","values":[136.334,118.753,132.061,144.754,121.864]},{"framework":"wasm-bindgen-v0.2.47-keyed","benchmark":"21_ready-memory","values":[1.0571517944335938,1.057159423828125,1.057159423828125,1.0571441650390625,1.0571441650390625]},{"framework":"wasm-bindgen-v0.2.47-keyed","benchmark":"22_run-memory","values":[1.5462188720703125,1.5462646484375,1.546234130859375,1.5461883544921875,1.5462722778320312]},{"framework":"wasm-bindgen-v0.2.47-keyed","benchmark":"23_update5-memory","values":[1.8680686950683594,1.8679542541503906,1.86798095703125,1.8680267333984375,1.8679962158203125]},{"framework":"wasm-bindgen-v0.2.47-keyed","benchmark":"24_run5-memory","values":[2.171539306640625,2.1714706420898438,2.1715240478515625,2.1715240478515625,2.171539306640625]},{"framework":"wasm-bindgen-v0.2.47-keyed","benchmark":"25_run-clear-memory","values":[2.316417694091797,2.3164329528808594,2.3163719177246094,2.3164024353027344,2.3164329528808594]},{"framework":"wasm-bindgen-v0.2.47-keyed","benchmark":"31_startup-ci","values":[1350.0680000000002,1350.408,1350.792,1350.6840000000002]},{"framework":"wasm-bindgen-v0.2.47-keyed","benchmark":"32_startup-bt","values":[18.476,20.4,20.539999999999992,20.959999999999997]},{"framework":"wasm-bindgen-v0.2.47-keyed","benchmark":"34_startup-totalbytes","values":[65.251953125,65.251953125,65.251953125,65.251953125]},{"framework":"yew-v0.16.2-keyed","benchmark":"01_run1k","values":[228.404,198.78,218.581,221.485,240.406]},{"framework":"yew-v0.16.2-keyed","benchmark":"02_replace1k","values":[86.72,85.166,86.531,82.068,82.298]},{"framework":"yew-v0.16.2-keyed","benchmark":"03_update10th1k_x16","values":[730.164,766.139,731.033,755.083,723.263]},{"framework":"yew-v0.16.2-keyed","benchmark":"04_select1k","values":[586.733,589.092,611.179,591.106,581.949]},{"framework":"yew-v0.16.2-keyed","benchmark":"05_swap1k","values":[607.884,636.007,606.045,602.641,626.062]},{"framework":"yew-v0.16.2-keyed","benchmark":"06_remove-one-1k","values":[153.362,151.125,153.889,149.862,147.31]},{"framework":"yew-v0.16.2-keyed","benchmark":"07_create10k","values":[2090.735,2152.461,2291.814,2253.821,2265.606]},{"framework":"yew-v0.16.2-keyed","benchmark":"08_create1k-after1k_x2","values":[449.371,481.864,467.01,582.564,599.141]},{"framework":"yew-v0.16.2-keyed","benchmark":"09_clear1k_x8","values":[379.549,386.233,372.865,373.012,366.459]},{"framework":"yew-v0.16.2-keyed","benchmark":"21_ready-memory","values":[1.0675430297851562,1.067535400390625,1.067535400390625,1.0675430297851562,1.0675811767578125]},{"framework":"yew-v0.16.2-keyed","benchmark":"22_run-memory","values":[2.141437530517578,2.141368865966797,2.1413536071777344,2.141376495361328,2.1413230895996094]},{"framework":"yew-v0.16.2-keyed","benchmark":"23_update5-memory","values":[2.143993377685547,2.143951416015625,2.143932342529297,2.1439247131347656,2.144023895263672]},{"framework":"yew-v0.16.2-keyed","benchmark":"24_run5-memory","values":[2.163043975830078,2.077831268310547,2.0777549743652344,2.0777549743652344,2.077739715576172]},{"framework":"yew-v0.16.2-keyed","benchmark":"25_run-clear-memory","values":[2.4068527221679688,2.4073028564453125,2.406890869140625,2.407318115234375,2.4072723388671875]},{"framework":"yew-v0.16.2-keyed","benchmark":"31_startup-ci","values":[1350.644,1350.996,1350.58,1350.324]},{"framework":"yew-v0.16.2-keyed","benchmark":"32_startup-bt","values":[32.251999999999995,30.119999999999994,27.37599999999999,30.35199999999999]},{"framework":"yew-v0.16.2-keyed","benchmark":"34_startup-totalbytes","values":[85.611328125,85.611328125,85.611328125,85.611328125]},{"framework":"yew-baseline-v0.16.2-keyed","benchmark":"01_run1k","values":[217.551,199.706,219.243,222.204,193.342]},{"framework":"yew-baseline-v0.16.2-keyed","benchmark":"02_replace1k","values":[77.833,76.875,85.177,83.126,92.603]},{"framework":"yew-baseline-v0.16.2-keyed","benchmark":"03_update10th1k_x16","values":[849.935,705.966,694.129,891.358,746.385]},{"framework":"yew-baseline-v0.16.2-keyed","benchmark":"04_select1k","values":[586.339,594.814,603.695,575.862,583.216]},{"framework":"yew-baseline-v0.16.2-keyed","benchmark":"05_swap1k","values":[613.011,619.557,629.485,602.918,622.681]},{"framework":"yew-baseline-v0.16.2-keyed","benchmark":"06_remove-one-1k","values":[148.837,155.306,153.872,156.057,146.042]},{"framework":"yew-baseline-v0.16.2-keyed","benchmark":"07_create10k","values":[2045.427,2174.11,2236.417,2229.609,2255.172]},{"framework":"yew-baseline-v0.16.2-keyed","benchmark":"08_create1k-after1k_x2","values":[513.862,489.675,475.759,470.244,469.735]},{"framework":"yew-baseline-v0.16.2-keyed","benchmark":"09_clear1k_x8","values":[360.767,385.571,380.572,440.869,439.504]},{"framework":"yew-baseline-v0.16.2-keyed","benchmark":"21_ready-memory","values":[1.0675277709960938,1.0675430297851562,1.0675430297851562,1.0675277709960938,1.0675506591796875]},{"framework":"yew-baseline-v0.16.2-keyed","benchmark":"22_run-memory","values":[2.141429901123047,2.141376495361328,2.141407012939453,2.141368865966797,2.1413841247558594]},{"framework":"yew-baseline-v0.16.2-keyed","benchmark":"23_update5-memory","values":[2.143951416015625,2.143993377685547,2.1439361572265625,2.1439743041992188,2.143951416015625]},{"framework":"yew-baseline-v0.16.2-keyed","benchmark":"24_run5-memory","values":[2.162822723388672,2.0782623291015625,2.0778236389160156,2.077808380126953,2.0777931213378906]},{"framework":"yew-baseline-v0.16.2-keyed","benchmark":"25_run-clear-memory","values":[2.4068679809570312,2.4069442749023438,2.4072647094726562,2.4072799682617188,2.4072799682617188]},{"framework":"yew-baseline-v0.16.2-keyed","benchmark":"31_startup-ci","values":[1350.772,1350.46,1350.512,1351.1080000000002]},{"framework":"yew-baseline-v0.16.2-keyed","benchmark":"32_startup-bt","values":[29.49199999999999,29.687999999999995,27.259999999999998,27.007999999999992]},{"framework":"yew-baseline-v0.16.2-keyed","benchmark":"34_startup-totalbytes","values":[85.6123046875,85.6123046875,85.6123046875,85.6123046875]}]

results_diff/src/main.rs

@@ -0,0 +1,60 @@
+use serde::Deserialize;
+use std::collections::BTreeMap;
+
+#[derive(Deserialize, Debug)]
+struct Result {
+    framework: String,
+    benchmark: String,
+    values: Vec<f64>,
+}
+
+#[derive(Default)]
+struct BenchmarkResults {
+    bindgen: f64,
+    baseline: f64,
+    update: f64,
+}
+
+fn main() {
+    let result_str = include_str!("../../webdriver-ts/results.json");
+    let results: Vec<Result> = serde_json::from_str(result_str).expect("failed to deserialize");
+    let mut benchmark_results: BTreeMap<String, BenchmarkResults> = BTreeMap::new();
+    for result in results.into_iter() {
+        let value_sum: f64 = result.values.iter().sum();
+        let value_count = result.values.len();
+        let avg_val = if value_count > 0 {
+            value_sum / value_count as f64
+        } else {
+            0f64
+        };
+
+        let mut entry = benchmark_results.entry(result.benchmark).or_default();
+        if result.framework.starts_with("wasm-bindgen") {
+            entry.bindgen = avg_val
+        } else if result.framework.starts_with("yew-baseline") {
+            entry.baseline = avg_val;
+        } else if result.framework.starts_with("yew") {
+            entry.update = avg_val;
+        }
+    }
+
+    let max_benchmark_name_len = benchmark_results.keys().map(|key| key.len()).max().unwrap();
+    let full_length = max_benchmark_name_len + 35 + 4 * 3 + 2; // 35: sum of columns defined below, 3: padding, 2: sign
+
+    println!("### Benchmark Report");
+    println!("```diff");
+    println!("@@ {:^1$} @@", "Performance Diff", full_length - 6);
+    println!();
+    println!("##{:>1$} | wasm-bindgen | baseline |  update |  diff ##", "", max_benchmark_name_len);
+    println!("{}", "#".repeat(full_length));
+    for (benchmark, results) in benchmark_results {
+        let diff = (results.update - results.baseline) / results.baseline; // TODO zero check
+        let sign = if diff < 0f64 { "+"} else { "-"};
+        print!("{} {:<2$} | ", sign, benchmark, max_benchmark_name_len);
+        print!("{:>1$} | ", format!("{:.2}", results.bindgen), 12); // 12: wasm-bindgen
+        print!("{:>1$} | ", format!("{:.2}", results.baseline), 8); // 8: baseline
+        print!("{:>1$} | ", format!("{:.2}", results.update), 7); // 7: f64 spacing
+        println!("{:>1$}", format!("{:+.2}%", 100f64 * diff), 8); // 8: pct spacing
+    }
+    println!("```");
+}

webdriver-ts/src/forkedBenchmarkRunner.ts

@@ -13,7 +13,6 @@ import * as R from 'ramda';
 let config:TConfig = defaultConfig;
 
 // necessary to launch without specifiying a path
-var chromedriver:any = require('chromedriver');
 var jStat:any = require('jstat').jStat;
 
 interface Timingresult {