Add --import-json to combine results from separate runs (fixes #830)

src/benchmark/benchmark_result.rs

@@ -1,20 +1,20 @@
 use std::collections::BTreeMap;
 
-use serde::Serialize;
+use serde::{Deserialize, Serialize};
 
 use crate::util::units::Second;
 
 /// Set of values that will be exported.
 // NOTE: `serde` is used for JSON serialization, but not for CSV serialization due to the
 // `parameters` map. Update `src/hyperfine/export/csv.rs` with new fields, as appropriate.
-#[derive(Debug, Default, Clone, Serialize, PartialEq)]
+#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq)]
 pub struct BenchmarkResult {
     /// The full command line of the program that is being benchmarked
     pub command: String,
 
     /// The full command line of the program that is being benchmarked, possibly including a list of
     /// parameters that were not used in the command line template.
-    #[serde(skip_serializing)]
+    #[serde(skip_serializing, default)]
     pub command_with_unused_parameters: String,
 
     /// The average run time
@@ -50,6 +50,6 @@ pub struct BenchmarkResult {
     pub exit_codes: Vec<Option<i32>>,
 
     /// Parameter values for this benchmark
-    #[serde(skip_serializing_if = "BTreeMap::is_empty")]
+    #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
     pub parameters: BTreeMap<String, String>,
 }

src/benchmark/scheduler.rs

@@ -31,6 +31,11 @@ impl<'a> Scheduler<'a> {
         }
     }
 
+    pub fn prepend_results(&mut self, mut imported_results: Vec<BenchmarkResult>) {
+        imported_results.append(&mut self.results);
+        self.results = imported_results;
+    }
+
     pub fn run_benchmarks(&mut self) -> Result<()> {
         let mut executor: Box<dyn Executor> = match self.options.executor_kind {
             ExecutorKind::Raw => Box::new(RawExecutor::new(self.options)),

src/cli.rs

@@ -298,6 +298,18 @@ fn build_command() -> Command {
                        the timing results for each individual run, use the JSON export format. \
                        The output time unit is always seconds."),
         )
+        .arg(
+            Arg::new("import-json")
+                .long("import-json")
+                .action(ArgAction::Set)
+                .value_name("FILE")
+                .value_hint(ValueHint::FilePath)
+                .help(
+                    "Import benchmark results from a previous JSON export (as produced by \
+                     --export-json) and include them in the summary and relative speed comparison \
+                     for this run.",
+                ),
+        )
         .arg(
             Arg::new("export-json")
                 .long("export-json")

src/error.rs

@@ -55,4 +55,6 @@ pub enum OptionsError<'a> {
     UnknownOutputPolicy(String),
     #[error("The file '{0}' specified as '--input' does not exist")]
     StdinDataFileDoesNotExist(String),
+    #[error("The file '{0}' specified as '--import-json' does not exist")]
+    ImportJsonFileDoesNotExist(String),
 }

src/export/json.rs

@@ -1,16 +1,19 @@
-use serde::*;
+use std::fs;
+use std::path::Path;
+
+use serde::{Deserialize, Serialize};
 use serde_json::to_vec_pretty;
 
 use super::Exporter;
 use crate::benchmark::benchmark_result::BenchmarkResult;
 use crate::options::SortOrder;
 use crate::util::units::Unit;
 
-use anyhow::Result;
+use anyhow::{Context, Result};
 
-#[derive(Serialize, Debug)]
-struct HyperfineSummary<'a> {
-    results: &'a [BenchmarkResult],
+#[derive(Serialize, Deserialize, Debug)]
+struct HyperfineSummary {
+    results: Vec<BenchmarkResult>,
 }
 
 #[derive(Default)]
@@ -23,11 +26,65 @@ impl Exporter for JsonExporter {
         _unit: Option<Unit>,
         _sort_order: SortOrder,
     ) -> Result<Vec<u8>> {
-        let mut output = to_vec_pretty(&HyperfineSummary { results });
+        let mut output = to_vec_pretty(&HyperfineSummary {
+            results: results.to_vec(),
+        });
         if let Ok(ref mut content) = output {
             content.push(b'\n');
         }
 
         Ok(output?)
     }
 }
+
+pub fn load_benchmark_results(path: &Path) -> Result<Vec<BenchmarkResult>> {
+    let content = fs::read(path)
+        .with_context(|| format!("Could not read JSON export file '{}'", path.display()))?;
+    let mut summary: HyperfineSummary = serde_json::from_slice(&content)
+        .with_context(|| format!("Could not parse JSON export file '{}'", path.display()))?;
+
+    for result in &mut summary.results {
+        if result.command_with_unused_parameters.is_empty() {
+            result.command_with_unused_parameters = result.command.clone();
+        }
+    }
+
+    Ok(summary.results)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::io::Write;
+
+    use tempfile::NamedTempFile;
+
+    #[test]
+    fn load_benchmark_results_restores_display_command() {
+        let exporter = JsonExporter::default();
+        let results = vec![BenchmarkResult {
+            command: "sleep 0.1".into(),
+            command_with_unused_parameters: "sleep 0.1".into(),
+            mean: 0.1,
+            stddev: None,
+            median: 0.1,
+            user: 0.0,
+            system: 0.0,
+            min: 0.1,
+            max: 0.1,
+            times: Some(vec![0.1]),
+            memory_usage_byte: None,
+            exit_codes: vec![Some(0)],
+            parameters: Default::default(),
+        }];
+
+        let json = exporter
+            .serialize(&results, None, SortOrder::Command)
+            .unwrap();
+        let mut file = NamedTempFile::new().unwrap();
+        file.write_all(&json).unwrap();
+
+        let loaded = load_benchmark_results(file.path()).unwrap();
+        assert_eq!(loaded, results);
+    }
+}

src/export/mod.rs

@@ -16,6 +16,8 @@ use self::json::JsonExporter;
 use self::markdown::MarkdownExporter;
 use self::orgmode::OrgmodeExporter;
 
+pub use self::json::load_benchmark_results;
+
 use crate::benchmark::benchmark_result::BenchmarkResult;
 use crate::options::SortOrder;
 use crate::util::units::Unit;

src/main.rs

@@ -8,6 +8,7 @@ use std::env;
 use benchmark::scheduler::Scheduler;
 use cli::get_cli_arguments;
 use command::Commands;
+use export::load_benchmark_results;
 use export::ExportManager;
 use options::Options;
 
@@ -43,6 +44,9 @@ fn run() -> Result<()> {
     options.validate_against_command_list(&commands)?;
 
     let mut scheduler = Scheduler::new(&commands, &options, &export_manager);
+    if let Some(path) = &options.import_json {
+        scheduler.prepend_results(load_benchmark_results(path)?);
+    }
     scheduler.run_benchmarks()?;
     scheduler.print_relative_speed_comparison();
     scheduler.final_export()?;

src/options.rs

@@ -246,6 +246,9 @@ pub struct Options {
 
     /// Which time unit to use when displaying results
     pub time_unit: Option<Unit>,
+
+    /// Previously exported results to include in comparison output
+    pub import_json: Option<PathBuf>,
 }
 
 impl Default for Options {
@@ -268,6 +271,7 @@ impl Default for Options {
             command_output_policies: vec![CommandOutputPolicy::Null],
             time_unit: None,
             command_input_policy: CommandInputPolicy::Null,
+            import_json: None,
         }
     }
 }
@@ -464,6 +468,16 @@ impl Options {
             CommandInputPolicy::Null
         };
 
+        options.import_json = matches.get_one::<String>("import-json").map(PathBuf::from);
+
+        if let Some(path) = &options.import_json {
+            if !path.exists() {
+                return Err(OptionsError::ImportJsonFileDoesNotExist(
+                    path.display().to_string(),
+                ));
+            }
+        }
+
         Ok(options)
     }
 

tests/integration_tests.rs

@@ -722,3 +722,41 @@ fn windows_quote_before_quote_args() {
         .assert()
         .success();
 }
+
+#[test]
+fn imports_json_export_for_relative_comparison() {
+    use tempfile::NamedTempFile;
+
+    let json_file = NamedTempFile::new().unwrap();
+    let json_path = json_file.path().to_path_buf();
+
+    hyperfine_debug()
+        .arg("--runs=1")
+        .arg(format!("--export-json={}", json_path.display()))
+        .arg("sleep 2.0")
+        .assert()
+        .success();
+
+    hyperfine_debug()
+        .arg("--runs=1")
+        .arg(format!("--import-json={}", json_path.display()))
+        .arg("sleep 1.0")
+        .assert()
+        .success()
+        .stdout(
+            predicate::str::contains("sleep 2.0")
+                .and(predicate::str::contains("2.00 times faster than sleep 2.0")),
+        );
+}
+
+#[test]
+fn fails_when_import_json_file_does_not_exist() {
+    hyperfine()
+        .arg("--import-json=does-not-exist.json")
+        .arg("echo test")
+        .assert()
+        .failure()
+        .stderr(predicate::str::contains(
+            "The file 'does-not-exist.json' specified as '--import-json' does not exist",
+        ));
+}