From c7e76cb846af78a4f09dc28cb97f2fb56c32bb44 Mon Sep 17 00:00:00 2001 From: Fraser Li Date: Fri, 17 Oct 2025 18:17:41 +1100 Subject: [PATCH] Implement reading parameters from files, see #813 --- src/cli.rs | 15 +++++ src/command.rs | 110 ++++++++++++++++++++++++++++----- tests/execution_order_tests.rs | 92 ++++++++++++++++++++++++++- 3 files changed, 198 insertions(+), 19 deletions(-) diff --git a/src/cli.rs b/src/cli.rs index eceabcb6c..34071e2f7 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -200,6 +200,21 @@ fn build_command() -> Command { possible parameter combinations.\n" ), ) + .arg( + Arg::new("parameter-list-file") + .long("parameter-list-file") + .action(ArgAction::Append) + .allow_hyphen_values(true) + .value_names(["VAR", "FILE"]) + .conflicts_with_all(["parameter-scan", "parameter-step-size"]) + .help( + "Perform benchmark runs for each line in the given FILE. Replaces the string \ + '{VAR}' in each command by the current line in FILE.\n\n Example: hyperfine \ + --parameter-list-file compiler compilers.txt '{compiler} -O2 main.cpp'\n\nThis \ + performs benchmarks for 'gcc -O2 main.cpp' and 'clang -O2 main.cpp' if the file \ + compilers.txt contains two lines, 'gcc' and 'clang'." + ), + ) .arg( Arg::new("shell") .long("shell") diff --git a/src/command.rs b/src/command.rs index 5d35cfaf7..dc31a8379 100644 --- a/src/command.rs +++ b/src/command.rs @@ -152,26 +152,14 @@ impl<'a> Commands<'a> { args, step_size, )?)) - } else if let Some(args) = matches.get_many::("parameter-list") { + } else if matches.contains_id("parameter-list") + || matches.contains_id("parameter-list-file") + { let command_names = command_names.map_or(vec![], |names| { names.map(|v| v.as_str()).collect::>() }); - let args: Vec<_> = args.map(|v| v.as_str()).collect::>(); - let param_names_and_values: Vec<(&str, Vec)> = args - .chunks_exact(2) - .map(|pair| { - let name = pair[0]; - let list_str = pair[1]; - (name, tokenize(list_str)) - }) - .collect(); - { - let duplicates = - Self::find_duplicates(param_names_and_values.iter().map(|(name, _)| *name)); - if !duplicates.is_empty() { - bail!("Duplicate parameter names: {}", &duplicates.join(", ")); - } - } + + let param_names_and_values = Self::get_parameter_lists(matches)?; let dimensions: Vec = std::iter::once(command_strings.len()) .chain( @@ -268,6 +256,69 @@ impl<'a> Commands<'a> { .collect() } + fn get_parameter_lists(matches: &'a ArgMatches) -> Result)>> { + let get_arg_pairs = |name| { + matches + .get_many::(name) + .unwrap_or_default() + .map(|a| a.as_str()) + .collect::>() + .chunks_exact(2) + .map(|pair| (pair[0], pair[1])) + .collect::>() + }; + + let cli_params = get_arg_pairs("parameter-list"); + let file_params = get_arg_pairs("parameter-list-file"); + + { + let cli_param_names = cli_params.iter().map(|(name, _)| *name); + let file_param_names = file_params.iter().map(|(name, _)| *name); + let duplicates = Self::find_duplicates(cli_param_names.chain(file_param_names)); + if !duplicates.is_empty() { + bail!("Duplicate parameter names: {}", &duplicates.join(", ")); + } + } + + let mut parameters = Vec::with_capacity(cli_params.len() + file_params.len()); + + let mut cli_params_it = cli_params.into_iter(); + let mut file_params_it = file_params.into_iter(); + + let mut cli_params_indices = matches + .indices_of("parameter-list") + .unwrap_or_default() + .peekable(); + let mut file_params_indices = matches + .indices_of("parameter-list-file") + .unwrap_or_default() + .peekable(); + + // Maintain ordering for instances of --parameter-list and --parameter-list-file + while cli_params_indices.peek().is_some() || file_params_indices.peek().is_some() { + let next_cli_index = cli_params_indices.peek(); + let next_file_index = file_params_indices.peek(); + if next_cli_index.is_some_and(|c| next_file_index.is_none_or(|f| c < f)) { + let (name, values) = cli_params_it.next().unwrap(); + parameters.push((name, tokenize(values))); + cli_params_indices.next(); + cli_params_indices.next(); + } else { + let (name, path) = file_params_it.next().unwrap(); + let values = std::fs::read_to_string(path) + .with_context(|| format!("Failed to read parameters from '{path}'"))? + .lines() + .map(ToOwned::to_owned) + .collect::>(); + parameters.push((name, values)); + file_params_indices.next(); + file_params_indices.next(); + } + } + + Ok(parameters) + } + fn build_parameter_scan_commands<'b, T: Numeric>( param_name: &'b str, param_min: T, @@ -449,6 +500,31 @@ fn test_build_parameter_list_commands() { assert_eq!(commands[1].get_command_line(), "echo 2"); } +#[test] +fn test_build_parameter_list_file_commands() { + use crate::cli::get_cli_arguments; + use std::io::Write; + + let mut parameters = tempfile::NamedTempFile::new().unwrap(); + parameters.write_all("1\n2\n".as_bytes()).unwrap(); + + let matches = get_cli_arguments(vec![ + "hyperfine", + "echo {foo}", + "--parameter-list-file", + "foo", + parameters.path().to_str().unwrap(), + "--command-name", + "name-{foo}", + ]); + let commands = Commands::from_cli_arguments(&matches).unwrap().0; + assert_eq!(commands.len(), 2); + assert_eq!(commands[0].get_name(), "name-1"); + assert_eq!(commands[1].get_name(), "name-2"); + assert_eq!(commands[0].get_command_line(), "echo 1"); + assert_eq!(commands[1].get_command_line(), "echo 2"); +} + #[test] fn test_build_parameter_scan_commands() { use crate::cli::get_cli_arguments; diff --git a/tests/execution_order_tests.rs b/tests/execution_order_tests.rs index 6ccc2e5ed..695b2946e 100644 --- a/tests/execution_order_tests.rs +++ b/tests/execution_order_tests.rs @@ -1,6 +1,7 @@ -use std::{fs::File, io::Read, path::PathBuf}; +use std::io::{Read, Write}; +use std::{fs::File, path::PathBuf}; -use tempfile::{tempdir, TempDir}; +use tempfile::{tempdir, NamedTempFile, TempDir}; mod common; use common::hyperfine; @@ -370,6 +371,93 @@ fn multiple_parameter_values() { .run(); } +#[test] +fn single_parameter_file() { + let mut numbers = NamedTempFile::new().unwrap(); + numbers.write_all("1\n2\n3\n".as_bytes()).unwrap(); + + ExecutionOrderTest::new() + .arg("--runs=2") + .arg("--parameter-list-file") + .arg("number") + .arg(numbers.path().to_str().unwrap()) + .command("command {number}") + .expect_output("command 1") + .expect_output("command 1") + .expect_output("command 2") + .expect_output("command 2") + .expect_output("command 3") + .expect_output("command 3") + .run(); +} + +#[test] +fn mutiple_parameter_files() { + let mut numbers = NamedTempFile::new().unwrap(); + numbers.write_all("1\n2\n3\n".as_bytes()).unwrap(); + + let mut letters = NamedTempFile::new().unwrap(); + letters.write_all("a\nb\n".as_bytes()).unwrap(); + + ExecutionOrderTest::new() + .arg("--runs=2") + .arg("--parameter-list-file") + .arg("number") + .arg(numbers.path().to_str().unwrap()) + .arg("--parameter-list-file") + .arg("letter") + .arg(letters.path().to_str().unwrap()) + .command("command {number} {letter}") + .expect_output("command 1 a") + .expect_output("command 1 a") + .expect_output("command 2 a") + .expect_output("command 2 a") + .expect_output("command 3 a") + .expect_output("command 3 a") + .expect_output("command 1 b") + .expect_output("command 1 b") + .expect_output("command 2 b") + .expect_output("command 2 b") + .expect_output("command 3 b") + .expect_output("command 3 b") + .run(); +} + +#[test] +fn parameter_values_and_files_interspersed() { + let mut numbers = NamedTempFile::new().unwrap(); + numbers.write_all("1\n2\n3\n".as_bytes()).unwrap(); + + let mut letters = NamedTempFile::new().unwrap(); + letters.write_all("a\nb\n".as_bytes()).unwrap(); + + ExecutionOrderTest::new() + .arg("--runs=1") + .arg("--parameter-list-file") + .arg("number") + .arg(numbers.path().to_str().unwrap()) + .arg("--parameter-list") + .arg("fruit") + .arg("apple,banana") + .arg("--parameter-list-file") + .arg("letter") + .arg(letters.path().to_str().unwrap()) + .command("command {number} {fruit} {letter}") + .expect_output("command 1 apple a") + .expect_output("command 2 apple a") + .expect_output("command 3 apple a") + .expect_output("command 1 banana a") + .expect_output("command 2 banana a") + .expect_output("command 3 banana a") + .expect_output("command 1 apple b") + .expect_output("command 2 apple b") + .expect_output("command 3 apple b") + .expect_output("command 1 banana b") + .expect_output("command 2 banana b") + .expect_output("command 3 banana b") + .run(); +} + #[test] fn reference_is_executed_first() { ExecutionOrderTest::new()