diff --git a/CHANGELOG.md b/CHANGELOG.md index fda110bee..aa395b41e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +# unreleased + +## Features + +- Add `--import-json ` option that loads benchmarks from a JSON file written by `--export-json` and includes them in the comparison output without re-running them, see #607 + # v1.20.0 ## Features diff --git a/README.md b/README.md index ff6020064..957644150 100644 --- a/README.md +++ b/README.md @@ -153,6 +153,20 @@ multiple benchmarks: | ![](doc/histogram.png) | ![](doc/whisker.png) | |---:|---:| +### Comparing against a previous run + +Use `--import-json ` to read a JSON file produced by a previous `--export-json` +run. The imported benchmarks are not re-executed: they appear in the relative speed +summary and in every export format alongside any commands you give on the command line. +This is handy when the baseline takes a long time to run, or lives on another machine, +or simply already exists from an earlier session: + +```sh +hyperfine 'old-binary args' --export-json baseline.json +hyperfine --import-json baseline.json 'new-binary args' +``` + +`--import-json` may be specified more than once to combine several saved files. ### Detailed benchmark flowchart diff --git a/src/benchmark/benchmark_result.rs b/src/benchmark/benchmark_result.rs index 287c73bef..0759eb08f 100644 --- a/src/benchmark/benchmark_result.rs +++ b/src/benchmark/benchmark_result.rs @@ -1,20 +1,20 @@ use std::collections::BTreeMap; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use crate::util::units::Second; /// Set of values that will be exported. // NOTE: `serde` is used for JSON serialization, but not for CSV serialization due to the // `parameters` map. Update `src/hyperfine/export/csv.rs` with new fields, as appropriate. -#[derive(Debug, Default, Clone, Serialize, PartialEq)] +#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq)] pub struct BenchmarkResult { /// The full command line of the program that is being benchmarked pub command: String, /// The full command line of the program that is being benchmarked, possibly including a list of /// parameters that were not used in the command line template. - #[serde(skip_serializing)] + #[serde(skip_serializing, default)] pub command_with_unused_parameters: String, /// The average run time @@ -39,17 +39,17 @@ pub struct BenchmarkResult { pub max: Second, /// All run time measurements - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(skip_serializing_if = "Option::is_none", default)] pub times: Option>, /// Maximum memory usage of the process, in bytes - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(skip_serializing_if = "Option::is_none", default)] pub memory_usage_byte: Option>, /// Exit codes of all command invocations pub exit_codes: Vec>, /// Parameter values for this benchmark - #[serde(skip_serializing_if = "BTreeMap::is_empty")] + #[serde(skip_serializing_if = "BTreeMap::is_empty", default)] pub parameters: BTreeMap, } diff --git a/src/benchmark/mod.rs b/src/benchmark/mod.rs index e3534a7bc..3e2c4225f 100644 --- a/src/benchmark/mod.rs +++ b/src/benchmark/mod.rs @@ -33,6 +33,7 @@ pub const MIN_EXECUTION_TIME: Second = 5e-3; pub struct Benchmark<'a> { number: usize, + display_number: usize, command: &'a Command<'a>, options: &'a Options, executor: &'a dyn Executor, @@ -47,12 +48,21 @@ impl<'a> Benchmark<'a> { ) -> Self { Benchmark { number, + display_number: number, command, options, executor, } } + /// Override the integer used for the `Benchmark N:` header. This is used + /// when imported benchmarks are listed before live ones, so the live ones + /// continue counting from the imported tail rather than restarting at 1. + pub fn with_display_number(mut self, display_number: usize) -> Self { + self.display_number = display_number; + self + } + /// Run setup, cleanup, or preparation commands fn run_intermediate_command( &self, @@ -143,7 +153,7 @@ impl<'a> Benchmark<'a> { println!( "{}{}: {}", "Benchmark ".bold(), - (self.number + 1).to_string().bold(), + (self.display_number + 1).to_string().bold(), self.command.get_name_with_unused_parameters(), ); } diff --git a/src/benchmark/scheduler.rs b/src/benchmark/scheduler.rs index 242dd0e7d..dc8aa59ec 100644 --- a/src/benchmark/scheduler.rs +++ b/src/benchmark/scheduler.rs @@ -17,6 +17,23 @@ pub struct Scheduler<'a> { results: Vec, } +/// Print a header for an imported benchmark. Mirrors the format used by +/// `Benchmark::run` so the imported entries blend in with the live ones. +fn print_imported_header(number: usize, result: &BenchmarkResult) { + let label = if result.command_with_unused_parameters.is_empty() { + result.command.as_str() + } else { + result.command_with_unused_parameters.as_str() + }; + println!( + "{}{}: {} {}", + "Benchmark ".bold(), + (number + 1).to_string().bold(), + label, + "(imported)".dimmed(), + ); +} + impl<'a> Scheduler<'a> { pub fn new( commands: &'a Commands, @@ -31,7 +48,29 @@ impl<'a> Scheduler<'a> { } } + /// Add benchmark results that were loaded from a previously-saved JSON file. + /// These are not re-run, but participate in the relative speed comparison + /// and in all configured exports as if they had just finished. + pub fn add_imported_results(&mut self, imported: Vec) { + if self.options.output_style != OutputStyleOption::Disabled { + for (offset, result) in imported.iter().enumerate() { + print_imported_header(self.results.len() + offset, result); + } + } + self.results.extend(imported); + } + pub fn run_benchmarks(&mut self) -> Result<()> { + // Preserve any pre-populated results (e.g. loaded via --import-json) so + // they share the same export pipeline as live benchmarks. + if !self.results.is_empty() { + self.export_manager.write_results(&self.results, true)?; + } + + if self.commands.iter().next().is_none() && self.options.reference_command.is_none() { + return Ok(()); + } + let mut executor: Box = match self.options.executor_kind { ExecutorKind::Raw => Box::new(RawExecutor::new(self.options)), ExecutorKind::Mock(ref shell) => Box::new(MockExecutor::new(shell.clone())), @@ -46,9 +85,13 @@ impl<'a> Scheduler<'a> { executor.calibrate()?; + let display_offset = self.results.len(); for (number, cmd) in reference.iter().chain(self.commands.iter()).enumerate() { - self.results - .push(Benchmark::new(number, cmd, self.options, &*executor).run()?); + self.results.push( + Benchmark::new(number, cmd, self.options, &*executor) + .with_display_number(number + display_offset) + .run()?, + ); // We export results after each individual benchmark, because // we would risk losing them if a later benchmark fails. diff --git a/src/cli.rs b/src/cli.rs index b12f6d34c..3e3d3a464 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -30,7 +30,7 @@ fn build_command() -> Command { The latter is only available if the shell is not explicitly disabled via \ '--shell=none'. If multiple commands are given, hyperfine will show a \ comparison of the respective runtimes.") - .required(true) + .required_unless_present("import-json") .action(ArgAction::Append) .value_hint(ValueHint::CommandString) .value_parser(NonEmptyStringValueParser::new()), @@ -325,6 +325,18 @@ fn build_command() -> Command { .help("Export the timing summary statistics as an Emacs org-mode table to the given FILE. \ The output time unit can be changed using the --time-unit option."), ) + .arg( + Arg::new("import-json") + .long("import-json") + .action(ArgAction::Append) + .num_args(1) + .value_name("FILE") + .value_hint(ValueHint::FilePath) + .help("Load benchmark results from a JSON file previously written by --export-json. \ + The imported benchmarks are not re-run, but appear alongside any commands given \ + on the command line in the relative speed comparison and in all export formats. \ + This option can be specified multiple times to import several files."), + ) .arg( Arg::new("show-output") .long("show-output") diff --git a/src/import/mod.rs b/src/import/mod.rs new file mode 100644 index 000000000..55508d646 --- /dev/null +++ b/src/import/mod.rs @@ -0,0 +1,146 @@ +use std::fs::File; +use std::io::BufReader; +use std::path::Path; + +use anyhow::{Context, Result}; +use serde::Deserialize; + +use crate::benchmark::benchmark_result::BenchmarkResult; + +/// Mirror of the structure written by `JsonExporter` so we can round-trip +/// previously-saved benchmark runs back into hyperfine. +#[derive(Deserialize)] +struct ImportedSummary { + results: Vec, +} + +/// Read a previously-saved JSON file and return the benchmark results inside it. +/// +/// The file is expected to follow the same schema written by `--export-json`, +/// i.e. a top-level object with a `"results"` array of benchmark objects. +pub fn load_results_from_json>(path: P) -> Result> { + let path = path.as_ref(); + let file = File::open(path) + .with_context(|| format!("Could not open import file '{}'", path.display()))?; + let reader = BufReader::new(file); + let summary: ImportedSummary = serde_json::from_reader(reader) + .with_context(|| format!("Failed to parse JSON from import file '{}'", path.display()))?; + + let mut results = summary.results; + for result in &mut results { + // The JSON schema does not carry `command_with_unused_parameters`, since + // `--export-json` skips it. Fall back to the command line itself so that + // the imported entry still has something sensible to display. + if result.command_with_unused_parameters.is_empty() { + result.command_with_unused_parameters = result.command.clone(); + } + } + Ok(results) +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use super::load_results_from_json; + use crate::benchmark::benchmark_result::BenchmarkResult; + + fn write_temp_json(contents: &str) -> std::path::PathBuf { + use std::io::Write; + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("results.json"); + let mut file = std::fs::File::create(&path).unwrap(); + file.write_all(contents.as_bytes()).unwrap(); + // Leak the tempdir so the file survives for the duration of the test. + let _ = dir.keep(); + path + } + + #[test] + fn round_trip_export_then_import() { + use serde_json::json; + + let original = vec![ + BenchmarkResult { + command: "sleep 0.1".into(), + command_with_unused_parameters: "sleep 0.1".into(), + mean: 0.1, + stddev: Some(0.01), + median: 0.1, + user: 0.05, + system: 0.02, + min: 0.09, + max: 0.11, + times: Some(vec![0.1, 0.1]), + memory_usage_byte: None, + exit_codes: vec![Some(0), Some(0)], + parameters: BTreeMap::new(), + }, + BenchmarkResult { + command: "sleep 0.2".into(), + command_with_unused_parameters: "sleep 0.2".into(), + mean: 0.2, + stddev: Some(0.02), + median: 0.2, + user: 0.1, + system: 0.05, + min: 0.18, + max: 0.22, + times: Some(vec![0.2, 0.2]), + memory_usage_byte: None, + exit_codes: vec![Some(0), Some(0)], + parameters: BTreeMap::new(), + }, + ]; + + // Re-emit via serde to confirm the schema written by `--export-json` + // round-trips back to a `BenchmarkResult` without information loss. + let summary = json!({ "results": &original }); + let path = write_temp_json(&summary.to_string()); + + let imported = load_results_from_json(&path).unwrap(); + assert_eq!(imported.len(), original.len()); + for (a, b) in imported.iter().zip(original.iter()) { + assert_eq!(a.command, b.command); + assert_eq!(a.mean, b.mean); + assert_eq!(a.stddev, b.stddev); + assert_eq!(a.median, b.median); + assert_eq!(a.times, b.times); + assert_eq!(a.exit_codes, b.exit_codes); + } + } + + #[test] + fn missing_optional_fields_are_tolerated() { + let path = write_temp_json( + r#"{ + "results": [ + { + "command": "echo a", + "mean": 0.5, + "stddev": null, + "median": 0.5, + "user": 0.0, + "system": 0.0, + "min": 0.5, + "max": 0.5, + "exit_codes": [0] + } + ] + }"#, + ); + let results = load_results_from_json(&path).unwrap(); + assert_eq!(results.len(), 1); + assert_eq!(results[0].command, "echo a"); + assert_eq!(results[0].command_with_unused_parameters, "echo a"); + assert_eq!(results[0].mean, 0.5); + assert!(results[0].times.is_none()); + } + + #[test] + fn missing_file_produces_a_helpful_error() { + let err = load_results_from_json("/no/such/file/hopefully.json").unwrap_err(); + let msg = format!("{err:#}"); + assert!(msg.contains("Could not open import file")); + } +} diff --git a/src/main.rs b/src/main.rs index 2a9750503..f924f4ad2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -19,6 +19,7 @@ pub mod cli; pub mod command; pub mod error; pub mod export; +pub mod import; pub mod options; pub mod outlier_detection; pub mod output; @@ -40,9 +41,22 @@ fn run() -> Result<()> { options.sort_order_exports, )?; + let imported_results = if let Some(paths) = cli_arguments.get_many::("import-json") { + let mut all = Vec::new(); + for path in paths { + all.extend(import::load_results_from_json(path)?); + } + all + } else { + Vec::new() + }; + options.validate_against_command_list(&commands)?; let mut scheduler = Scheduler::new(&commands, &options, &export_manager); + if !imported_results.is_empty() { + scheduler.add_imported_results(imported_results); + } scheduler.run_benchmarks()?; scheduler.print_relative_speed_comparison(); scheduler.final_export()?; diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 2de3a5049..6b9f0c6fa 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -705,6 +705,131 @@ fn speed_comparison_sort_order() { )); } +#[test] +fn import_json_brings_saved_runs_into_comparison() { + use tempfile::tempdir; + + let tempdir = tempdir().unwrap(); + let saved = tempdir.path().join("saved.json"); + + // Save a run for `sleep 1.0` and `sleep 2.0`. + hyperfine_debug() + .arg("--style=none") + .arg("--export-json") + .arg(&saved) + .arg("sleep 1.0") + .arg("sleep 2.0") + .assert() + .success(); + + // Import that file alongside a fresh run for `sleep 1.5`. The imported + // entries should appear in the headers and in the relative speed summary + // without being re-executed. + hyperfine_debug() + .arg("--style=basic") + .arg("--import-json") + .arg(&saved) + .arg("sleep 1.5") + .assert() + .success() + .stdout( + predicate::str::contains("Benchmark 1: sleep 1.0 (imported)") + .and(predicate::str::contains( + "Benchmark 2: sleep 2.0 (imported)", + )) + .and(predicate::str::contains("Benchmark 3: sleep 1.5")) + .and(predicate::str::contains( + "1.50 ± 0.00 times faster than sleep 1.5", + )) + .and(predicate::str::contains( + "2.00 ± 0.00 times faster than sleep 2.0", + )), + ); +} + +#[test] +fn import_json_works_without_any_command() { + use tempfile::tempdir; + + let tempdir = tempdir().unwrap(); + let saved = tempdir.path().join("saved.json"); + + hyperfine_debug() + .arg("--style=none") + .arg("--export-json") + .arg(&saved) + .arg("sleep 1.0") + .arg("sleep 2.0") + .assert() + .success(); + + hyperfine_debug() + .arg("--style=basic") + .arg("--import-json") + .arg(&saved) + .assert() + .success() + .stdout( + predicate::str::contains("Benchmark 1: sleep 1.0 (imported)") + .and(predicate::str::contains( + "Benchmark 2: sleep 2.0 (imported)", + )) + .and(predicate::str::contains( + "2.00 ± 0.00 times faster than sleep 2.0", + )), + ); +} + +#[test] +fn import_json_can_be_specified_multiple_times() { + use tempfile::tempdir; + + let tempdir = tempdir().unwrap(); + let first = tempdir.path().join("first.json"); + let second = tempdir.path().join("second.json"); + + hyperfine_debug() + .arg("--style=none") + .arg("--export-json") + .arg(&first) + .arg("sleep 1.0") + .assert() + .success(); + + hyperfine_debug() + .arg("--style=none") + .arg("--export-json") + .arg(&second) + .arg("sleep 2.0") + .assert() + .success(); + + hyperfine_debug() + .arg("--style=basic") + .arg("--import-json") + .arg(&first) + .arg("--import-json") + .arg(&second) + .assert() + .success() + .stdout( + predicate::str::contains("Benchmark 1: sleep 1.0 (imported)").and( + predicate::str::contains("Benchmark 2: sleep 2.0 (imported)"), + ), + ); +} + +#[test] +fn import_json_fails_for_missing_file() { + hyperfine() + .arg("--import-json") + .arg("nonexistent_b5d9574198b7e4b12a71fa4747c0a577.json") + .arg("echo a") + .assert() + .failure() + .stderr(predicate::str::contains("Could not open import file")); +} + #[cfg(windows)] #[test] fn windows_quote_args() {