diff --git a/Cargo.lock b/Cargo.lock index b81c6670f87cc..a4bbdceeb3990 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -74,7 +74,7 @@ checksum = "5b515e82c8468ddb6ff8db21c78a5997442f113fd8471fd5b2261b2602dd0c67" dependencies = [ "num_enum", "serde", - "strum", + "strum 0.25.0", ] [[package]] @@ -732,6 +732,15 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" +[[package]] +name = "ansi_term" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +dependencies = [ + "winapi", +] + [[package]] name = "anstream" version = "0.6.15" @@ -1676,7 +1685,7 @@ version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" dependencies = [ - "generic-array", + "generic-array 0.14.7", ] [[package]] @@ -1685,6 +1694,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7bc6d6292be3a19e6379786dac800f551e5865a5bb51ebbe3064ab80433f403" dependencies = [ + "generic-array 0.14.7", "ff", "group", "pairing", @@ -2239,8 +2249,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b34115915337defe99b2aff5c2ce6771e5fbc4079f4b506301f5cf394c8452f7" dependencies = [ "crossterm", - "strum", - "strum_macros", + "strum 0.25.0", + "strum_macros 0.25.3", "unicode-width", ] @@ -2475,7 +2485,7 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ - "generic-array", + "generic-array 0.14.7", "typenum", ] @@ -2689,7 +2699,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" dependencies = [ - "generic-array", + "generic-array 0.14.7", ] [[package]] @@ -2821,6 +2831,18 @@ dependencies = [ "spki", ] +[[package]] +name = "educe" +version = "0.4.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f0042ff8246a363dbe77d2ceedb073339e85a804b9a47636c6e016a9a32c05f" +dependencies = [ + "enum-ordinalize", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "either" version = "1.13.0" @@ -2849,7 +2871,7 @@ dependencies = [ "crypto-bigint", "digest 0.10.7", "ff", - "generic-array", + "generic-array 0.14.7", "group", "pem-rfc7468", "pkcs8", @@ -3069,7 +3091,7 @@ dependencies = [ "const-hex", "elliptic-curve", "ethabi", - "generic-array", + "generic-array 0.14.7", "k256", "num_enum", "once_cell", @@ -3314,6 +3336,7 @@ dependencies = [ "foundry-config", "foundry-debugger", "foundry-evm", + "foundry-evm-mutator", "foundry-linking", "foundry-test-utils", "foundry-wallets", @@ -3605,7 +3628,7 @@ dependencies = [ "regex", "serde", "strsim", - "strum", + "strum 0.25.0", "tempfile", "tokio", "tracing", @@ -3644,6 +3667,9 @@ dependencies = [ "foundry-compilers", "foundry-config", "foundry-macros", + "glob", + "globset", + "indicatif", "num-format", "once_cell", "reqwest", @@ -3960,6 +3986,20 @@ dependencies = [ "tracing", ] +[[package]] +name = "foundry-evm-mutator" +version = "0.2.0" +dependencies = [ + "alloy-json-abi", + "alloy-primitives", + "eyre", + "foundry-cli", + "foundry-common", + "foundry-compilers", + "gambit", + "itertools 0.11.0", +] + [[package]] name = "foundry-evm-traces" version = "0.2.0" @@ -4270,6 +4310,32 @@ dependencies = [ "windows 0.58.0", ] +[[package]] +name = "gambit" +version = "0.2.1" +source = "git+https://github.com/samparsky/gambit?rev=dbdc911a450f2ada494016dbbc522245b5f3dc94#dbdc911a450f2ada494016dbbc522245b5f3dc94" +dependencies = [ + "ansi_term", + "clap", + "clap_complete", + "csv", + "env_logger", + "itertools 0.12.0", + "log", + "project-root", + "rand 0.8.5", + "rand_chacha 0.3.1", + "rand_pcg", + "regex", + "scanner-rust", + "serde", + "serde_json", + "similar", + "strum 0.24.1", + "strum_macros 0.24.3", + "tempfile", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -4281,6 +4347,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "generic-array" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe739944a5406424e080edccb6add95685130b9f160d5407c639c7df0c5836b0" +dependencies = [ + "typenum", +] + [[package]] name = "getrandom" version = "0.2.15" @@ -5111,7 +5186,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" dependencies = [ - "generic-array", + "generic-array 0.14.7", ] [[package]] @@ -6634,6 +6709,12 @@ dependencies = [ "tokio-stream", ] +[[package]] +name = "project-root" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bccbff07d5ed689c4087d20d7307a52ab6141edeedf487c3876a55b86cf63df" + [[package]] name = "proptest" version = "1.5.0" @@ -6853,6 +6934,15 @@ dependencies = [ "getrandom", ] +[[package]] +name = "rand_pcg" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59cad018caf63deb318e5a4586d99a24424a364f40f1e5778c29aca23f4fc73e" +dependencies = [ + "rand_core 0.6.4", +] + [[package]] name = "rand_xorshift" version = "0.3.0" @@ -7554,7 +7644,7 @@ checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" dependencies = [ "base16ct", "der", - "generic-array", + "generic-array 0.14.7", "pkcs8", "subtle", "zeroize", @@ -8126,7 +8216,20 @@ version = "0.26.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" dependencies = [ - "strum_macros", + "strum_macros 0.25.3", +] + +[[package]] +name = "strum_macros" +version = "0.24.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn 1.0.109", ] [[package]] @@ -9014,6 +9117,12 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" +[[package]] +name = "utf8-width" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86bd8d4e895da8537e5315b8254664e6b769c4ff3db18321b297a1e7004392e3" + [[package]] name = "utf8parse" version = "0.2.2" diff --git a/Cargo.toml b/Cargo.toml index e78d0049231fa..0d1a9c71800e5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,7 @@ members = [ "crates/forge/", "crates/macros/", "crates/test-utils/", + "crates/evm/mutator/" ] resolver = "2" @@ -153,6 +154,8 @@ foundry-evm-fuzz = { path = "crates/evm/fuzz" } foundry-evm-traces = { path = "crates/evm/traces" } foundry-macros = { path = "crates/macros" } foundry-test-utils = { path = "crates/test-utils" } + +foundry-evm-mutator = { path = "crates/evm/mutator" } foundry-wallets = { path = "crates/wallets" } foundry-linking = { path = "crates/linking" } diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml index 5c843979a97e6..a004043aad22a 100644 --- a/crates/common/Cargo.toml +++ b/crates/common/Cargo.toml @@ -63,6 +63,7 @@ tracing.workspace = true url.workspace = true walkdir.workspace = true yansi.workspace = true +indicatif = "0.17" [dev-dependencies] foundry-macros.workspace = true diff --git a/crates/common/src/term.rs b/crates/common/src/term.rs index 925456e4c65f2..705dedc8f740e 100644 --- a/crates/common/src/term.rs +++ b/crates/common/src/term.rs @@ -196,6 +196,148 @@ pub fn with_spinner_reporter(f: impl FnOnce() -> T) -> T { report::with_scoped(&reporter, f) } +/// A spinner used for reporting in Mutation tests +/// +/// This reporter will prefix messages with a spinning cursor +#[derive(Debug)] +pub struct MutatorSpinnerReporter { + /// The sender to the spinner thread. + sender: mpsc::Sender +} + +impl Drop for MutatorSpinnerReporter { + fn drop(&mut self) { + let (tx, rx) = mpsc::channel(); + if self.sender.send(SpinnerMsg::Shutdown(tx)).is_ok() { + let _ = rx.recv(); + } + } +} + +impl MutatorSpinnerReporter { + /// Spawns the [`Spinner`] on a new thread + /// + /// The spinner's message will be updated via the `reporter` events + /// + /// On drop the channel will disconnect and the thread will terminate + pub fn spawn(message: String) -> Self { + let (sender, rx) = mpsc::channel::(); + + std::thread::Builder::new() + .name("mutator".into()) + .spawn(move || { + let mut spinner = Spinner::new(message); + loop { + spinner.tick(); + match rx.try_recv() { + Ok(SpinnerMsg::Msg(msg)) => { + spinner.message(msg); + // new line so past messages are not overwritten + println!(); + } + Ok(SpinnerMsg::Shutdown(ack)) => { + // end with a newline + println!(); + let _ = ack.send(()); + break + } + Err(TryRecvError::Disconnected) => break, + Err(TryRecvError::Empty) => thread::sleep(Duration::from_millis(100)), + } + } + }) + .expect("failed to spawn thread"); + + MutatorSpinnerReporter { sender } + } + + + // fn send_msg(&self, msg: impl Into) { + // let _ = self.sender.send(SpinnerMsg::Msg(msg.into())); + // } +} + + +enum ProgressMsg { + Increment, + Shutdown(mpsc::Sender<()>), +} + +/// A progress bar used for reporting progess +/// +/// Spins up a dedicated thread. +#[derive(Debug, Clone)] +pub struct ProgressReporter { + /// The sender to the spinner thread. + sender: mpsc::Sender +} + +impl ProgressReporter { + /// Spawns a progress bar on a new thread + /// + /// + /// On drop the channel will disconnect and the thread will terminate + pub fn spawn(label: String, size: usize) -> Self { + let (sender, rx) = mpsc::channel::(); + + std::thread::Builder::new() + .name("mutator".into()) + .spawn(move || { + let pb = indicatif::ProgressBar::new(size as u64); + let mut template = + "{spinner:.green} [{elapsed_precise}] [{wide_bar:.cyan/blue}] {pos}/{len} ".to_string(); + template += &label; + template += " ({eta})"; + pb.set_style( + indicatif::ProgressStyle::with_template(&template) + .unwrap() + .with_key("eta", eta_key) + .progress_chars("#>-"), + ); + pb.set_position(0); + + let mut current_index = 0; + loop { + pb.tick(); + match rx.try_recv() { + Ok(ProgressMsg::Increment) => { + current_index += 1; + // new line so past messages are not overwritten + pb.set_position(current_index); + } + Ok(ProgressMsg::Shutdown(ack)) => { + pb.finish_and_clear(); + let _ = ack.send(()); + break + } + Err(TryRecvError::Disconnected) => break, + Err(TryRecvError::Empty) => thread::sleep(Duration::from_millis(100)), + } + } + }) + .expect("failed to spawn thread"); + + Self { sender } + } + + /// Increment the progress bar + pub fn increment(&self) { + let _ = self.sender.send(ProgressMsg::Increment); + } + + /// Finish and clear progress bar + pub fn finish_and_clear(&self) { + let (tx, rx) = mpsc::channel(); + if self.sender.send(ProgressMsg::Shutdown(tx)).is_ok() { + let _ = rx.recv(); + } + } +} + +fn eta_key(state: &indicatif::ProgressState, f: &mut dyn std::fmt::Write) { + write!(f, "{:.1}s", state.eta().as_secs_f64()).unwrap() +} + #[macro_export] /// Displays warnings on the cli macro_rules! cli_warn { diff --git a/crates/common/src/traits.rs b/crates/common/src/traits.rs index f5f3ea14ce460..2bf1c38c42db4 100644 --- a/crates/common/src/traits.rs +++ b/crates/common/src/traits.rs @@ -240,3 +240,17 @@ impl ErrorExt for T { alloy_sol_types::Revert::from(self.to_string()).abi_encode().into() } } + +/// Extension trait for matching functions +pub trait FunctionFilter { + /// Returns whether the function should be included + fn matches_function(&self, function_name: impl AsRef) -> bool; +} + +/// Extension trait for matching functions +pub trait ContractFilter { + /// Returns whether the contract should be included + fn matches_contract(&self, contract_name: impl AsRef) -> bool; + /// Returns a contract with the given path should be included + fn matches_path(&self, path: &Path) -> bool; +} \ No newline at end of file diff --git a/crates/config/src/lib.rs b/crates/config/src/lib.rs index ca7226e18da42..f7f7b08a8f412 100644 --- a/crates/config/src/lib.rs +++ b/crates/config/src/lib.rs @@ -115,6 +115,10 @@ use vyper::VyperConfig; mod bind_json; use bind_json::BindJsonConfig; +mod mutate; +pub use mutate::MutateConfig; + + /// Foundry configuration /// /// # Defaults @@ -453,6 +457,9 @@ pub struct Config { /// Warnings gathered when loading the Config. See [`WarningsProvider`] for more information #[serde(rename = "__warnings", default, skip_serializing)] pub warnings: Vec, + + /// Configures the Mutate test setup + pub mutate: MutateConfig, /// PRIVATE: This structure may grow, As such, constructing this structure should /// _always_ be done using a public constructor or update syntax: @@ -2143,6 +2150,7 @@ impl Default for Config { extra_args: vec![], eof_version: None, alphanet: false, + mutate: Default::default(), _non_exhaustive: (), } } diff --git a/crates/config/src/mutate.rs b/crates/config/src/mutate.rs new file mode 100644 index 0000000000000..e29c4aeceaa3e --- /dev/null +++ b/crates/config/src/mutate.rs @@ -0,0 +1,142 @@ +//! Configuration specific to the `forge mutate` command + +use serde::{Deserialize, Serialize}; +use std::{path::PathBuf, str::FromStr}; +use crate::{RegexWrapper, from_opt_glob, FuzzConfig}; +use std::time::Duration; + +/// Contains the mutation test config +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct MutateConfig { + /// path to where mutation artifacts should be written to + #[serde(default = "default_out_path")] + pub out: PathBuf, + + /// Flag to write out mutants + #[serde(default = "default_export")] + pub export: bool, + + /// Only run mutations for functions matching the specified regex pattern. + #[serde(rename = "match_function")] + pub function_pattern: Option, + + /// Only run mutations on functions that do not match the specified regex pattern. + #[serde(rename = "no_match_function")] + pub function_pattern_inverse: Option, + + /// Only run mutations on functions in contracts matching the specified regex pattern. + #[serde(rename = "match_contract")] + pub contract_pattern: Option, + + /// Only run mutations in contracts that do not match the specified regex pattern. + #[serde(rename = "no_match_contract")] + pub contract_pattern_inverse: Option, + + /// Only run mutations on source files matching the specified glob pattern. + #[serde(rename = "match_path", with = "from_opt_glob")] + pub path_pattern: Option, + + /// Only run mutations on source files that do not match the specified glob pattern. + #[serde(rename = "no_match_path", with = "from_opt_glob")] + pub path_pattern_inverse: Option, + + /// Only run test functions matching the specified regex pattern. + #[serde(rename = "match_test")] + pub test_pattern: Option, + + /// Only run test functions that do not match the specified regex pattern. + #[serde(rename = "no_match_test")] + pub test_pattern_inverse: Option, + + /// Only run tests in contracts matching the specified regex pattern. + #[serde(rename = "match_test_contract")] + pub test_contract_pattern: Option, + + /// Only run tests in contracts that do not match the specified regex pattern. + #[serde(rename = "no_match_test_contract")] + pub test_contract_pattern_inverse: Option, + + /// Only run tests in source files matching the specified glob pattern. + #[serde(rename = "match_test_path", with = "from_opt_glob")] + pub test_path_pattern: Option, + + /// Only run tests in source files that do not match the specified glob pattern. + #[serde(rename = "no_match_test_path", with = "from_opt_glob")] + pub test_path_pattern_inverse: Option, + + /// Timeout for tests it helps exit long running test + #[serde(default = "default_test_timeout")] + pub test_timeout: Duration, + + /// Fuzz configuration + pub fuzz: FuzzConfig, + + /// Number of mutants to execute concurrently + /// + /// This should be configured conservatively because of "Too Many Files Open Error" as + /// we use join_all to run tasks in concurrently + #[serde(default = "default_parallel")] + pub parallel: usize, + + /// Max Timeout + /// + /// Maximum number of tests allowed to timeout, this is required because a test run + /// can be long depending on the mutation. This leads to memory consumption per each + /// mutant test that runs for a long time. + /// We configure this value here to put a bound on the possible memory leak for this + /// + /// This is required because we can't cancel a thread + /// + /// 16 * 32 MB (EVM default memory limit) = 512 MB + #[serde(default = "default_maximum_timeout_test")] + pub maximum_timeout_test: usize, +} + + +impl Default for MutateConfig { + fn default() -> Self { + Self { + out: default_out_path(), + export: default_export(), + function_pattern: None, + function_pattern_inverse: None, + contract_pattern: None, + contract_pattern_inverse: None, + path_pattern: None, + path_pattern_inverse: None, + test_pattern: None, + test_pattern_inverse: None, + test_contract_pattern: None, + test_contract_pattern_inverse: None, + test_path_pattern: None, + test_path_pattern_inverse: None, + test_timeout: default_test_timeout(), + fuzz: FuzzConfig { + runs: 1, + ..Default::default() + }, + parallel: default_parallel(), + maximum_timeout_test: default_maximum_timeout_test(), + } + } +} + +fn default_out_path() -> PathBuf { + PathBuf::from_str(".gambit").unwrap() +} + +fn default_export() -> bool { + false +} + +fn default_parallel() -> usize { + 16 +} + +fn default_maximum_timeout_test() -> usize { + 16 +} + +fn default_test_timeout() -> Duration { + Duration::from_millis(100) +} \ No newline at end of file diff --git a/crates/evm/mutator/Cargo.toml b/crates/evm/mutator/Cargo.toml new file mode 100644 index 0000000000000..73be89b0e1df6 --- /dev/null +++ b/crates/evm/mutator/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "foundry-evm-mutator" +description = "Mutation testing framework using `gambit`" + +version.workspace = true +edition.workspace = true +rust-version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true +exclude.workspace = true + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +foundry-compilers = { workspace = true, features = ["full"] } +eyre.workspace = true +itertools.workspace = true +foundry-common.workspace = true +alloy-json-abi.workspace = true +foundry-cli.workspace = true +gambit = { git = "https://github.com/samparsky/gambit", rev = "dbdc911a450f2ada494016dbbc522245b5f3dc94" } +alloy-primitives = { workspace = true, features = ["serde"] } diff --git a/crates/evm/mutator/src/lib.rs b/crates/evm/mutator/src/lib.rs new file mode 100644 index 0000000000000..cc64a439dfdfa --- /dev/null +++ b/crates/evm/mutator/src/lib.rs @@ -0,0 +1,236 @@ +use alloy_json_abi::{Function, JsonAbi as Abi}; +use alloy_primitives::Bytes; +use eyre::{eyre, Result}; +use foundry_cli::utils::FoundryPathExt; +use foundry_common::{ContractFilter, FunctionFilter, TestFunctionExt}; +use foundry_compilers::{ + remappings::RelativeRemapping, Artifact, ArtifactId, ArtifactOutput, ProjectCompileOutput, +}; +use gambit::{run_mutate, MutateParams}; +use itertools::Itertools; +use std::{ + collections::{BTreeMap, HashMap}, + path::PathBuf, +}; + +pub use gambit::Mutant; + +/// Array of artifact ids, abi and bytecode +pub type GambitArtifacts = Vec<(ArtifactId, Abi, Bytes)>; + +#[derive(Debug, Clone)] +pub struct MutatorConfigBuilder { + solc: PathBuf, + solc_allow_paths: Vec, + solc_include_paths: Vec, + solc_remappings: Vec, + solc_optimize: bool, +} + +impl MutatorConfigBuilder { + pub fn new( + solc: PathBuf, + solc_optimize: bool, + solc_allow_paths: Vec, + solc_include_paths: Vec, + solc_remappings: Vec, + ) -> Self { + Self { solc, solc_allow_paths, solc_include_paths, solc_remappings, solc_optimize } + } + + pub fn build( + self, + src_folder_root: PathBuf, + output: ProjectCompileOutput, + ) -> Result { + // Converts the compiled output into artifactId and abi + // It does not include files with .t.sol extension + let artifacts: Vec<(ArtifactId, Abi, Bytes)> = output + .into_artifacts() + .filter_map(|(id, c)| match (id.source.as_path().is_sol_test(), c.into_parts()) { + (false, (Some(abi), Some(bytecode), _)) => Some((id, abi, bytecode)), + _ => None, + }) + .collect::>(); + + let solc = self.solc.to_str().ok_or(eyre!("failed to decode solc root"))?; + let solc_allow_paths: Vec = self + .solc_allow_paths + .into_iter() + .filter_map(|x| x.to_str().map(|x| x.to_string())) + .collect(); + let solc_include_paths: String = self + .solc_include_paths + .into_iter() + .filter_map(|x| x.to_str().map(|x| x.to_string())) + .join(","); + let solc_remappings: Vec = + self.solc_remappings.into_iter().map(|x| x.to_string()).collect(); + let source_root = src_folder_root.to_str().ok_or(eyre!("failed to decode source root"))?; + + Ok(Mutator::new( + artifacts, + source_root.to_owned(), + solc.to_owned(), + solc_allow_paths, + solc_include_paths, + solc_remappings, + self.solc_optimize, + )) + } +} + +#[derive(Debug, Clone)] +pub struct Mutator { + src_root: PathBuf, + artifacts: GambitArtifacts, + default_mutate_params: MutateParams, +} + +impl Mutator { + pub fn new( + artifacts: GambitArtifacts, + source_root: String, + solc: String, + solc_allow_paths: Vec, + solc_include_paths: String, + solc_remappings: Vec, + solc_optimize: bool, + ) -> Self { + // create mutate params here + let src_root = PathBuf::from(&source_root); + let default_mutate_params = MutateParams { + json: None, + filename: None, + num_mutants: None, + random_seed: false, + seed: 0, + outdir: None, + sourceroot: Some(source_root.into()), + mutations: None, + no_export: true, + no_overwrite: false, + solc: solc.into(), + solc_optimize, + functions: None, + contract: None, + solc_base_path: None, + solc_allow_paths: Some(solc_allow_paths.into()), + solc_include_path: Some(solc_include_paths.into()), + solc_remappings: Some(solc_remappings.into()), + skip_validate: false, + }; + + Self { src_root, artifacts, default_mutate_params } + } + + /// Returns the number of matching functions + pub fn matching_function_count(&self, filter: &A) -> usize { + self.filtered_functions(filter).count() + } + + /// Returns the name of the functions to generate Mutants + pub fn get_artifact_functions<'a, A>( + &'a self, + filter: &'a A, + abi: &'a Abi, + ) -> impl Iterator + 'a + where + A: ContractFilter + FunctionFilter, + { + abi.functions() + .filter_map(|func| filter.matches_function(&func.name).then_some(func.name.clone())) + } + + /// Returns an iterator of functions matching filter + pub fn filtered_functions<'a, A>(&'a self, filter: &'a A) -> impl Iterator + where + A: ContractFilter + FunctionFilter, + { + self.matching_artifacts(filter).flat_map(|(_, abi, _)| abi.functions()) + } + + /// Returns an iterator of function names matching filter + pub fn get_function_names<'a, A>(&'a self, filter: &'a A) -> impl Iterator + 'a + where + A: ContractFilter + FunctionFilter, + { + self.filtered_functions(filter) + .filter_map(|func| filter.matches_function(&func.name).then_some(&func.name)) + } + + /// Returns mutation relevant artifacts matching the filter + pub fn matching_artifacts<'a, A>( + &'a self, + filter: &'a A, + ) -> impl Iterator + where + A: ContractFilter + FunctionFilter, + { + self.artifacts.iter().filter(|(id, abi, _)| { + id.source.starts_with(&self.src_root) && + !id.source.as_path().is_sol_test() && + filter.matches_path(&id.source) && + filter.matches_contract(&id.name) && + abi.functions().any(|func| filter.matches_function(&func.name)) + }) + } + + /// Returns all matching functions grouped by contract + /// grouped by file (file -> contract -> functions) + pub fn list( + &self, + filter: &A, + ) -> BTreeMap>> { + self.matching_artifacts(filter) + .map(|(id, abi, _)| { + let source = id.source.as_path().display().to_string(); + let name = id.name.clone(); + let functions = abi + .functions() + .filter(|func| !func.name.is_test()) + .filter(|func| filter.matches_function(func.name.clone())) + .map(|func| func.name.clone()) + .collect::>(); + + (source, name, functions) + }) + .fold(BTreeMap::new(), |mut acc, (source, name, functions)| { + acc.entry(source).or_default().insert(name, functions); + acc + }) + } + + /// Run mutation on contract functions that match configured filters + /// @TODO we should support ability to disable writing out artifacts + pub fn run_mutate( + self, + _: bool, + default_out_dir: PathBuf, + filter: A, + ) -> Result>> + where + A: ContractFilter + FunctionFilter, + { + let mutant_params = self + .matching_artifacts(&filter) + .map(|(id, abi, _)| { + let mut current_mutate_params = self.default_mutate_params.clone(); + let outdir = default_out_dir.join(id.name.clone()); + current_mutate_params.outdir = outdir.to_str().map(|x| x.to_owned()); + current_mutate_params.functions = + Some(self.get_artifact_functions(&filter, abi).collect_vec()); + current_mutate_params.filename = + Some(String::from(id.source.to_str().expect("failed run mutate filename"))); + current_mutate_params.contract = get_contract_name(&id.name); + current_mutate_params + }) + .collect_vec(); + + run_mutate(mutant_params).map_err(|err| eyre!("{:?}", err)) + } +} + +fn get_contract_name(name: &str) -> Option { + name.split(".").nth(0).map(|x| x.to_owned()) +} \ No newline at end of file diff --git a/crates/forge/Cargo.toml b/crates/forge/Cargo.toml index eb3e94406a080..b60e278aadad9 100644 --- a/crates/forge/Cargo.toml +++ b/crates/forge/Cargo.toml @@ -36,6 +36,8 @@ foundry-linking.workspace = true ethers-contract-abigen = { workspace = true, features = ["providers"] } +foundry-evm-mutator.workspace = true + revm-inspectors.workspace = true comfy-table.workspace = true @@ -111,6 +113,9 @@ soldeer.workspace = true [target.'cfg(unix)'.dependencies] tikv-jemallocator = { workspace = true, optional = true } +# temp file +tempfile = { version = "3.8.0" } + [dev-dependencies] anvil.workspace = true foundry-test-utils.workspace = true diff --git a/crates/forge/bin/cmd/mod.rs b/crates/forge/bin/cmd/mod.rs index ff63fa7cbc0eb..12ee84e3d18b7 100644 --- a/crates/forge/bin/cmd/mod.rs +++ b/crates/forge/bin/cmd/mod.rs @@ -66,3 +66,4 @@ pub mod test; pub mod tree; pub mod update; pub mod watch; +pub mod mutate; \ No newline at end of file diff --git a/crates/forge/bin/cmd/mutate/filter.rs b/crates/forge/bin/cmd/mutate/filter.rs new file mode 100644 index 0000000000000..a62eea37c9d48 --- /dev/null +++ b/crates/forge/bin/cmd/mutate/filter.rs @@ -0,0 +1,289 @@ +use crate::cmd::test::{FilterArgs, ProjectPathsAwareFilter}; +use clap::Parser; +use foundry_cli::utils::FoundryPathExt; +use foundry_common::{ + glob::GlobMatcher, + traits::{FunctionFilter, TestFunctionExt}, + ContractFilter, +}; +use foundry_compilers::{FileFilter, ProjectPathsConfig}; +use foundry_config::Config; +use std::{fmt, path::Path}; + +/// The filter to use during mutation testing. +/// +/// See also `FileFilter`. +#[derive(Clone, Parser)] +#[clap(next_help_heading = "Mutation Test filtering")] +pub struct MutateFilterArgs { + /// Only run mutations on functions matching the specified regex pattern. + #[clap(long = "match-function", visible_alias = "mf", value_name = "REGEX")] + pub function_pattern: Option, + + /// Only run mutations on functions that do not match the specified regex pattern. + #[clap(long = "no-match-function", visible_alias = "nmf", value_name = "REGEX")] + pub function_pattern_inverse: Option, + + /// Only run mutations on functions in contracts matching the specified regex pattern. + #[clap(long = "match-contract", visible_alias = "mc", value_name = "REGEX")] + pub contract_pattern: Option, + + /// Only run mutations in contracts that do not match the specified regex pattern. + #[clap(long = "no-match-contract", visible_alias = "nmc", value_name = "REGEX")] + pub contract_pattern_inverse: Option, + + /// Only run mutations on source files matching the specified glob pattern. + #[clap(long = "match-path", visible_alias = "mp", value_name = "GLOB")] + pub path_pattern: Option, + + /// Only run mutations on source files that do not match the specified glob pattern. + #[clap( + name = "no-match-path", + long = "no-match-path", + visible_alias = "nmp", + value_name = "GLOB" + )] + pub path_pattern_inverse: Option, + + /// Only run test functions matching the specified regex pattern. + #[clap(long = "match-test")] + pub test_pattern: Option, + + /// Only run test functions that do not match the specified regex pattern. + #[clap(long = "no-match-test")] + pub test_pattern_inverse: Option, + + /// Only run tests in contracts matching the specified regex pattern. + #[clap(long = "match-test-contract")] + pub test_contract_pattern: Option, + + /// Only run tests in contracts that do not match the specified regex pattern. + #[clap(long = "no-match-test-contract")] + pub test_contract_pattern_inverse: Option, + + /// Only run tests in source files matching the specified glob pattern. + #[clap(long = "match-test-path", value_name = "GLOB")] + pub test_path_pattern: Option, + + /// Only run tests in source files that do not match the specified glob pattern. + #[clap(long = "no-match-test-path", value_name = "GLOB")] + pub test_path_pattern_inverse: Option, +} + +impl MutateFilterArgs { + /// Merges the set filter globs with the config's values + /// Returns mutate and test filters + pub fn merge_with_config( + &self, + config: &Config, + ) -> (MutationProjectPathsAwareFilter, ProjectPathsAwareFilter) { + let mut filter = self.clone(); + if filter.function_pattern.is_none() { + filter.function_pattern = config.mutate.function_pattern.clone().map(|p| p.into()); + } + if filter.function_pattern_inverse.is_none() { + filter.function_pattern_inverse = + config.mutate.function_pattern_inverse.clone().map(|p| p.into()); + } + if filter.contract_pattern.is_none() { + filter.contract_pattern = config.mutate.contract_pattern.clone().map(|p| p.into()); + } + if filter.contract_pattern_inverse.is_none() { + filter.contract_pattern_inverse = + config.mutate.contract_pattern_inverse.clone().map(|p| p.into()); + } + if filter.path_pattern.is_none() { + filter.path_pattern = config.mutate.path_pattern.clone().map(Into::into); + } + if filter.path_pattern_inverse.is_none() { + filter.path_pattern_inverse = + config.mutate.path_pattern_inverse.clone().map(Into::into); + } + + // Parse test filter + let test_filter: FilterArgs = FilterArgs { + test_pattern: filter.test_pattern.clone(), + test_pattern_inverse: filter.test_pattern_inverse.clone(), + contract_pattern: filter.test_contract_pattern.clone(), + contract_pattern_inverse: filter.test_contract_pattern_inverse.clone(), + path_pattern: filter.test_path_pattern.clone(), + path_pattern_inverse: filter.test_path_pattern_inverse.clone(), + }; + let test_paths_aware_filter = test_filter.merge_with_mutate_config(&config); + + ( + MutationProjectPathsAwareFilter { args_filter: filter, paths: config.project_paths() }, + test_paths_aware_filter, + ) + } +} + +impl fmt::Debug for MutateFilterArgs { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("MutateFilterArgs") + .field("match-function", &self.function_pattern.as_ref().map(|r| r.as_str())) + .field("no-match-function", &self.function_pattern_inverse.as_ref().map(|r| r.as_str())) + .field("match-contract", &self.contract_pattern.as_ref().map(|r| r.as_str())) + .field("no-match-contract", &self.contract_pattern_inverse.as_ref().map(|r| r.as_str())) + .field("match-path", &self.path_pattern.as_ref().map(|g| g.as_str())) + .field("no-match-path", &self.path_pattern_inverse.as_ref().map(|g| g.as_str())) + .field("match-test", &self.test_pattern.as_ref().map(|r| r.as_str())) + .field("no-match-test", &self.test_path_pattern_inverse.as_ref().map(|r| r.as_str())) + .field("match-test-contract", &self.test_contract_pattern.as_ref().map(|r| r.as_str())) + .field( + "no-match-test-contract", + &self.test_contract_pattern_inverse.as_ref().map(|r| r.as_str()), + ) + .field("match-test-path", &self.test_path_pattern.as_ref().map(|g| g.as_str())) + .field( + "no-match-test-path", + &self.test_path_pattern_inverse.as_ref().map(|g| g.as_str()), + ) + .finish_non_exhaustive() + } +} + +impl FileFilter for MutateFilterArgs { + /// Returns true if the file regex pattern match the `file` + /// + /// If no file regex is set this returns true if the file ends with `.t.sol`, see + /// [FoundryPathExr::is_sol_test()] + fn is_match(&self, file: &Path) -> bool { + if let Some(file) = file.as_os_str().to_str() { + if let Some(ref glob) = self.path_pattern { + return glob.is_match(file); + } + if let Some(ref glob) = self.path_pattern_inverse { + return !glob.is_match(file); + } + } + file.is_sol_test() + } +} + +impl ContractFilter for MutateFilterArgs { + fn matches_contract(&self, contract_name: impl AsRef) -> bool { + let mut ok = true; + let contract_name = contract_name.as_ref(); + if let Some(re) = &self.contract_pattern { + ok &= re.is_match(contract_name); + } + if let Some(re) = &self.contract_pattern_inverse { + ok &= !re.is_match(contract_name); + } + ok + } + + fn matches_path(&self, path: &Path) -> bool { + let Some(path) = path.to_str() else { + return false; + }; + + let mut ok = true; + if let Some(ref glob) = self.path_pattern { + ok &= glob.is_match(path); + } + if let Some(ref glob) = self.path_pattern_inverse { + ok &= !glob.is_match(path); + } + ok + } +} + +impl FunctionFilter for MutateFilterArgs { + fn matches_function(&self, function_name: impl AsRef) -> bool { + let mut ok = true; + let function_name = function_name.as_ref(); + + if let Some(re) = &self.function_pattern { + ok &= re.is_match(function_name); + } + + if let Some(re) = &self.function_pattern_inverse { + ok &= !re.is_match(function_name); + } + + ok &= !function_name.is_test(); + + ok + } +} + +impl fmt::Display for MutateFilterArgs { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut patterns = Vec::new(); + if let Some(ref p) = self.function_pattern { + patterns.push(format!("\tmatch-function: `{}`", p.as_str())); + } + if let Some(ref p) = self.function_pattern_inverse { + patterns.push(format!("\tno-match-function: `{}`", p.as_str())); + } + if let Some(ref p) = self.contract_pattern { + patterns.push(format!("\tmatch-contract: `{}`", p.as_str())); + } + if let Some(ref p) = self.contract_pattern_inverse { + patterns.push(format!("\tno-match-contract: `{}`", p.as_str())); + } + if let Some(ref p) = self.path_pattern { + patterns.push(format!("\tmatch-path: `{}`", p.as_str())); + } + if let Some(ref p) = self.path_pattern_inverse { + patterns.push(format!("\tno-match-path: `{}`", p.as_str())); + } + write!(f, "{}", patterns.join("\n")) + } +} + +/// A filter that combines all command line arguments and the paths of the current projects +#[derive(Debug, Clone)] +pub struct MutationProjectPathsAwareFilter { + args_filter: MutateFilterArgs, + paths: ProjectPathsConfig, +} + +impl MutationProjectPathsAwareFilter { + /// Returns the CLI arguments + pub fn args(&self) -> &MutateFilterArgs { + &self.args_filter + } + + /// Returns the CLI arguments mutably + pub fn args_mut(&mut self) -> &mut MutateFilterArgs { + &mut self.args_filter + } +} + +impl FileFilter for MutationProjectPathsAwareFilter { + /// Returns true if the file regex pattern match the `file` + /// + /// If no file regex is set this returns true if the file ends with `.t.sol`, see + /// [FoundryPathExr::is_sol_test()] + fn is_match(&self, file: &Path) -> bool { + self.args_filter.is_match(file) + } +} + +impl ContractFilter for MutationProjectPathsAwareFilter { + fn matches_contract(&self, contract_name: impl AsRef) -> bool { + self.args_filter.matches_contract(contract_name) + } + + fn matches_path(&self, path: &Path) -> bool { + // we don't want to test files that belong to a library + self.args_filter.matches_path(path) && + !self.paths.has_library_ancestor(path) && + !Path::new(path).starts_with(&self.paths.tests) + } +} + +impl FunctionFilter for MutationProjectPathsAwareFilter { + fn matches_function(&self, function_name: impl AsRef) -> bool { + self.args_filter.matches_function(function_name) + } +} + +impl fmt::Display for MutationProjectPathsAwareFilter { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.args_filter.fmt(f) + } +} diff --git a/crates/forge/bin/cmd/mutate/mod.rs b/crates/forge/bin/cmd/mutate/mod.rs new file mode 100644 index 0000000000000..08630da3cfcf0 --- /dev/null +++ b/crates/forge/bin/cmd/mutate/mod.rs @@ -0,0 +1,624 @@ +use super::{ + install, + test::{ProjectPathsAwareFilter, TestOutcome}, +}; +use alloy_primitives::U256; +use crate::cmd::mutate::summary::{ + MutantTestResult, MutationTestOutcome, MutationTestSuiteResult, MutationTestSummaryReporter, +}; +use clap::Parser; +use eyre::{eyre, Result}; +use forge::{ + inspectors::CheatsConfig, result::SuiteResult, revm::primitives::Env, + MultiContractRunnerBuilder, TestOptions, TestOptionsBuilder, +}; +use foundry_cli::{ + opts::CoreBuildArgs, + utils::{self, LoadConfig}, +}; +use foundry_common::{ + compile::ProjectCompiler, + evm::EvmArgs, + shell::{self}, + term::{MutatorSpinnerReporter, ProgressReporter}, +}; +use foundry_compilers::{ + project_util::{copy_dir, TempProject}, + Project, ProjectCompileOutput, +}; +use foundry_config::{ + figment, + figment::{ + value::{Dict, Map}, + Metadata, Profile, Provider, + }, + get_available_profiles, Config, +}; +use foundry_evm::{backend::Backend, opts::EvmOpts}; +use foundry_evm_mutator::{Mutant, MutatorConfigBuilder}; +use futures::future::{join_all, try_join_all}; +use itertools::Itertools; +use std::{ + collections::{BTreeMap, HashMap as StdHashMap}, + fs, + path::PathBuf, + sync::mpsc::channel, + time::{Duration, Instant}, +}; + +mod filter; +pub use filter::*; +mod summary; +pub use summary::*; + +// Loads project's figment and merges the build cli arguments into it +foundry_config::merge_impl_figment_convert!(MutateTestArgs, opts, evm_opts); + +/// CLI arguments for `forge mutate`. +#[derive(Debug, Clone, Parser)] +#[clap(next_help_heading = "Mutation Test options")] +pub struct MutateTestArgs { + /// Output mutate results in JSON format. + #[clap(long, short, help_heading = "Display options")] + json: bool, + + #[clap(flatten)] + filter: MutateFilterArgs, + + /// Exit with code 0 even if a test fails. + #[clap(long, env = "FORGE_ALLOW_FAILURE")] + allow_failure: bool, + + /// Stop running mutation tests after the first surviving mutation + #[clap(long)] + pub fail_fast: bool, + + /// List all matching functions instead of running them + #[clap(long, short, help_heading = "Display options")] + list: bool, + + /// Set seed used to generate randomness during your fuzz runs. + #[clap(long)] + pub fuzz_seed: Option, + + #[clap(long, env = "FOUNDRY_FUZZ_RUNS", value_name = "RUNS")] + pub fuzz_runs: Option, + + #[clap(flatten)] + evm_opts: EvmArgs, + + #[clap(flatten)] + opts: CoreBuildArgs, + + /// Export generated mutants to a directory + #[clap(long, default_value_t = false)] + pub export: bool, + + /// Print mutation test summary table + #[clap(long, help_heading = "Display options", default_value_t = false)] + pub summary: bool, + + /// Print detailed mutation test summary table + #[clap(long, help_heading = "Display options")] + pub detailed: bool, +} + +impl MutateTestArgs { + /// Returns the flattened [`CoreBuildArgs`]. + pub fn build_args(&self) -> &CoreBuildArgs { + &self.opts + } + + pub async fn run(self) -> Result<()> { + trace!(target: "forge::mutate", "executing mutation command"); + shell::set_shell(shell::Shell::from_args(self.opts.silent, self.json))?; + println!( + "{}", + "[⠆] Starting Mutation Test. Go grab a cup of coffee ☕, it's going to take a while" + ); + + let mutation_test_outcome = self.execute_mutation_test().await?; + println!(); + mutation_test_outcome.ensure_ok()?; + + Ok(()) + } + + /// Executes mutation test for the project + /// + /// This will trigger the build process and run tests that match the configured filter. + /// On success mutants will get be generated for functions matching configured filter. + /// On success tests matching configured filter will be executed for all mutants. + /// + /// Returns the mutation test results for all matching functions + pub async fn execute_mutation_test(self) -> Result { + let (mut config, evm_opts) = self.load_config_and_evm_opts_emit_warnings()?; + // Fetch project mutate and test filter + let (mutate_filter, test_filter) = self.filter(&config); + + // Set up the project + let mut project = config.project()?; + + // install missing dependencies + if install::install_missing_dependencies(&mut config, self.build_args().silent) && + config.auto_detect_remappings + { + // need to re-configure here to also catch additional remappings + config = self.load_config(); + project = config.project()?; + } + + let (test_outcome, output) = + self.ensure_valid_project(&project, &config, &evm_opts, test_filter.clone()).await?; + + // Ensure test outcome is ok, exit if any test is failing + if test_outcome.failures().count() > 0 { + test_outcome.ensure_ok()?; + } + + let (mutants_output, mutants_len) = + self.execute_mutation(&project, mutate_filter, output, &config)?; + + println!("[⠆] Testing Mutants..."); + let env = evm_opts.evm_env().await?; + let test_backend = Backend::spawn(evm_opts.get_fork(&config, env.clone())).await; + let progress_bar_reporter = ProgressReporter::spawn("Mutants".into(), mutants_len); + + let mut mutant_test_suite_results: BTreeMap = + BTreeMap::new(); + + let mutation_project_root = project.root(); + let mutant_fuzz_runs = self.fuzz_runs.unwrap_or(0) as u32; + let mutant_fuzz_seed = self.fuzz_seed.clone(); + for (contract_out_dir, contract_mutants) in mutants_output.into_iter() { + let mut mutant_test_statuses: Vec<(Duration, MutantTestStatus)> = + Vec::with_capacity(contract_mutants.len()); + let contract_mutants_start = Instant::now(); + + // a file can have hundreds of mutants depending on design + // we chunk there to prevent huge memory consumption + // join_all which launches all the futures and polls + let mut contract_mutants_iterator = contract_mutants.chunks(config.mutate.parallel); + + + while let Some(mutant_chunks) = contract_mutants_iterator.next() { + let mutant_data_iterator = mutant_chunks.iter().map(|mutant| { + ( + mutation_project_root.clone(), + mutant.source.filename_as_str(), + mutant.as_source_string().expect("Failed to read a file"), + ) + }); + + // we compile the projects here + let mutant_project_and_compile_output: Vec<_> = + try_join_all(mutant_data_iterator.map(|(root, file_name, mutant_contents)| { + tokio::task::spawn_blocking(move || { + setup_and_compile_mutant( + root, + file_name, + mutant_contents, + mutant_fuzz_runs.clone(), + mutant_fuzz_seed.clone() + ) + }) + })) + .await? + .into_iter() + .filter_map(|x| x.ok()) + .collect(); + + let test_output = join_all(mutant_project_and_compile_output.into_iter().map( + |(temp_project, mutant_compile_output, mutant_config)| { + test_mutant( + test_backend.clone(), + progress_bar_reporter.clone(), + test_filter.clone(), + temp_project, + mutant_config, + mutant_compile_output, + &evm_opts, + env.clone(), + ) + }, + )) + .await; + + mutant_test_statuses.extend(test_output); + } + + let mutant_test_results: Vec<_> = + std::iter::zip(contract_mutants, mutant_test_statuses) + .map(|(mutant, (duration, status))| { + MutantTestResult::new(duration, mutant, status) + }) + .collect_vec(); + + let has_survivors = mutant_test_results.iter().any(|result| result.survived()); + let contract_name = contract_out_dir + .split(std::path::MAIN_SEPARATOR_STR) + .nth(1) + .ok_or(eyre!("Failed to parse contract name"))?; + + mutant_test_suite_results.insert( + contract_name.to_string(), + MutationTestSuiteResult::new(contract_mutants_start.elapsed(), mutant_test_results), + ); + + // Exit if fail fast is configured + if self.fail_fast && has_survivors { + break; + } + + // Exit if number of maximum number of timeout tests is reached + if mutant_test_suite_results.values().flat_map(|result| result.timeout()).count() >= + config.mutate.maximum_timeout_test + { + break; + } + } + + // finish progress bar and clear it + progress_bar_reporter.finish_and_clear(); + + if self.json { + println!( + "{}", + serde_json::to_string_pretty( + &mutant_test_suite_results + .values() + .flat_map(|x| x.mutation_test_results()) + .collect::>() + )? + ); + std::process::exit(0); + } + + let mutation_test_outcome = + MutationTestOutcome::new(self.allow_failure, mutant_test_suite_results); + + if self.summary { + let mut reporter = MutationTestSummaryReporter::new(self.detailed); + reporter.print_summary(&mutation_test_outcome); + } + + println!("{}", mutation_test_outcome.summary()); + + Ok(mutation_test_outcome) + } + + /// Performs mutation on the project contracts + pub fn execute_mutation( + &self, + project: &Project, + mutate_filter: MutationProjectPathsAwareFilter, + output: ProjectCompileOutput, + config: &Config, + ) -> Result<(StdHashMap>, usize)> { + trace!(target: "forge::mutate", "running gambit"); + + let mutator = MutatorConfigBuilder::new( + project.solc.solc.clone(), + config.optimizer, + project.allowed_paths.paths().map(|x| x.to_owned()).collect_vec(), + project.include_paths.paths().map(|x| x.to_owned()).collect(), + config.remappings.clone(), + ) + .build(config.src.clone(), output)?; + + if mutator.matching_function_count(&mutate_filter) == 0 { + println!("\nNo functions match the provided pattern"); + println!("{}", mutate_filter.to_string()); + // Try to suggest a function when there's no match + if let Some(ref function_pattern) = mutate_filter.args().function_pattern { + let function_name = function_pattern.as_str(); + let candidates = mutator.get_function_names(&mutate_filter); + if let Some(suggestion) = utils::did_you_mean(function_name, candidates).pop() { + println!("\nDid you mean `{suggestion}`?"); + } + std::process::exit(0); + } + } + + if self.list { + println!(); + + let results = mutator.list(&mutate_filter); + for (source, mutants) in results.iter() { + for (name, functions) in mutants.iter() { + println!("{}:{}", source, name); + println!("\t{}", functions.join(" \n\t")); + } + } + std::process::exit(0); + } + + let spinner = MutatorSpinnerReporter::spawn("Generating Mutants...".into()); + + let now = Instant::now(); + // init spinner + let mutants_output = + mutator.run_mutate(self.export, config.mutate.out.clone(), mutate_filter.clone())?; + + let elapsed = now.elapsed(); + + drop(spinner); + + trace!(target: "forge::mutate", "finished running gambit"); + + let mutants_len = mutants_output.iter().flat_map(|(_, v)| v).count(); + + println!("Generated {} mutants in {:.2?}", mutants_len, elapsed); + + Ok((mutants_output, mutants_len)) + } + + /// Compiles and Tests the project to ensure no failing tests + pub async fn ensure_valid_project( + &self, + project: &Project, + config: &Config, + evm_opts: &EvmOpts, + filter: ProjectPathsAwareFilter, + ) -> Result<(TestOutcome, ProjectCompileOutput)> { + let compiler = ProjectCompiler::default(); + let output = compiler.compile(&project)?; + // Create test options from general project settings + // and compiler output + let start = Instant::now(); + let test_reporter = MutatorSpinnerReporter::spawn("Running Project Tests...".into()); + let project_root = &project.paths.root; + let toml = config.get_config_path(); + let profiles = get_available_profiles(toml)?; + let env = evm_opts.evm_env().await?; + + let test_options: TestOptions = TestOptionsBuilder::default() + .fuzz(config.mutate.fuzz) + .invariant(config.invariant) + .profiles(profiles) + .build(&output, project_root)?; + + let runner_builder = MultiContractRunnerBuilder::default() + .set_debug(false) + .initial_balance(evm_opts.initial_balance) + .evm_spec(config.evm_spec_id()) + .sender(evm_opts.sender) + .with_fork(evm_opts.get_fork(&config, env.clone())) + .with_cheats_config(CheatsConfig::new(&config, evm_opts.clone())) + .with_test_options(test_options.clone()); + + let mut runner = runner_builder.clone().build( + project_root, + output.clone(), + env.clone(), + evm_opts.clone(), + )?; + + if runner.matching_test_function_count(&filter) == 0 { + let filter_str = filter.to_string(); + if filter_str.is_empty() { + println!( + "\nNo tests found in project! Forge looks for functions that starts with `test`." + ); + } else { + println!("\nNo tests match the provided pattern:"); + println!("{filter_str}"); + // Try to suggest a test when there's no match + if let Some(ref test_pattern) = filter.args().test_pattern { + let test_name = test_pattern.as_str(); + let candidates = runner.get_tests(&filter); + if let Some(suggestion) = utils::did_you_mean(test_name, candidates).pop() { + println!("\nDid you mean `{suggestion}`?"); + } + } + } + } + + let mut results = BTreeMap::new(); + // Set up test reporter channel + let (tx, rx) = channel::<(String, SuiteResult)>(); + + // Run tests + let handle = + tokio::task::spawn(async move { runner.test(&filter, tx, test_options).await }); + + 'outer: for (contract_name, suite_result) in rx { + results.insert(contract_name.clone(), suite_result.clone()); + if suite_result.failures().count() > 0 { + break 'outer + } + } + let _results = handle.await?; + + // stop reporter + drop(test_reporter); + + println!("Finished running project tests in {:2?}", start.elapsed()); + Ok((TestOutcome::new(results, false), output)) + } + + /// Returns the flattened [`MutateFilterArgs`] arguments merged with [`Config`]. + pub fn filter( + &self, + config: &Config, + ) -> (MutationProjectPathsAwareFilter, ProjectPathsAwareFilter) { + self.filter.merge_with_config(config) + } +} + +impl Provider for MutateTestArgs { + fn metadata(&self) -> Metadata { + Metadata::named("Mutation Test: Args ") + } + + fn data(&self) -> Result, figment::Error> { + let mut dict = Dict::default(); + + let mut mutate_dict = Dict::default(); + // Override the fuzz and invariants run + // We do not want fuzz and invariant tests to run once so the test + // setup is faster. + let mut fuzz_dict = Dict::default(); + if let Some(fuzz_seed) = self.fuzz_seed { + fuzz_dict.insert("seed".to_string(), fuzz_seed.to_string().into()); + } + if let Some(fuzz_runs) = self.fuzz_runs { + fuzz_dict.insert("runs".to_string(), fuzz_runs.into()); + } + mutate_dict.insert("fuzz".to_string(), fuzz_dict.into()); + // insert into config + dict.insert("mutate".to_string(), mutate_dict.into()); + + let mut invariant_dict = Dict::default(); + invariant_dict.insert("runs".to_string(), 0.into()); + dict.insert("invariant".to_string(), invariant_dict.into()); + + Ok(Map::from([(Config::selected_profile(), dict)])) + } +} + +/// Creates a temp project from source project and compiles the project +pub fn setup_and_compile_mutant( + mutation_project_root: PathBuf, + mutant_file: String, + mutant_contents: String, + fuzz_runs: u32, + fuzz_seed: Option +) -> Result<(TempProject, ProjectCompileOutput, Config)> { + trace!(target: "forge::mutate", "setting up and compiling mutant"); + + let start = Instant::now(); + + // we do not support hardhat style testing + let temp_project = TempProject::dapptools()?; + let temp_project_root = temp_project.root(); + + // copy project source code to temp dir + copy_dir(mutation_project_root, temp_project_root)?; + + // load config for this temp project + let mut config = Config::load_with_root(temp_project_root); + // appends the root dir to the config folder variables + // it's important + config = config.canonic_at(temp_project_root); + // override fuzz and invariant runs + config.mutate.fuzz.runs = fuzz_runs; + config.mutate.fuzz.seed = fuzz_seed; + config.invariant.runs = 0; + + let mutant_file_path = temp_project_root.join(mutant_file); + // Write Mutant contents to file in temp_directory + fs::write(&mutant_file_path.as_path(), mutant_contents)?; + + debug!( + duration = ?start.elapsed(), + "compilation times", + ); + + let mut project = config.project()?; + project.set_solc_jobs(4); + + let compile_output = project.compile().map_err(|_| eyre!("compilation failed"))?; + + trace!(target: "forge::mutate", "finishing setting up and compiling mutant"); + + Ok((temp_project, compile_output, config)) +} + +/// Runs mutation test for a mutation temp project. +/// returns on the first failed test suite +pub async fn test_mutant( + db: Backend, + progress_bar: ProgressReporter, + filter: ProjectPathsAwareFilter, + temp_project: TempProject, + config: Config, + output: ProjectCompileOutput, + evm_opts: &EvmOpts, + env: Env, +) -> (Duration, MutantTestStatus) { + trace!(target: "forge::mutate", "testing mutant"); + + let start = Instant::now(); + + let project_root = temp_project.root(); + let toml = config.get_config_path(); + let profiles = get_available_profiles(toml).expect("Failed to get profiles"); + + let test_options: TestOptions = TestOptionsBuilder::default() + .fuzz(config.mutate.fuzz) + .invariant(config.invariant) + .profiles(profiles) + .build(&output, project_root) + .expect("Failed to setup test options"); + + let runner_builder = MultiContractRunnerBuilder::default() + .set_debug(false) + .initial_balance(evm_opts.initial_balance) + .evm_spec(config.evm_spec_id()) + .sender(evm_opts.sender) + .with_fork(evm_opts.get_fork(&config, env.clone())) + .with_cheats_config(CheatsConfig::new(&config, evm_opts.clone())) + .with_test_options(test_options.clone()); + + let mut runner = runner_builder + .build(project_root, output.clone(), env, evm_opts.clone()) + .expect("Failed to build test runner"); + + // We use a thread and recv_timeout here because Gambit generates + // valid solidity grammar mutants that leads to very long running + // execution in REVM due to large amount of gas available. + // + // An example of a mutant generated is below, the test will take forever to end + // because it's an infinite loop + // + // unchecked { + // UnaryOperatorMutation(`++` |==> `~`) of: `for (uint256 i = 0; i < owners.length; ++i) + // for (uint256 i = 0; i < owners.length; ~i) { + // balances[i] = balanceOf[owners[i]][ids[i]]; + // } + // } + // + // + // mpsc channel for reporting test results + let (tx, rx) = channel::<(String, SuiteResult)>(); + let _ = tokio::task::spawn_blocking(move || { + // We create ThreadPool here because it's possible to have + // a long running test that attaches itself to the global rayon ThreadPool + // This would prevent other rayon tasks from executing leading to a deadlock + // Creating a pool means we can isolate this and prevent it from affecting + // other tasks. + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(1) + .build() + .expect("Failed to setup thread pool "); + + pool.install(|| runner.test_with_backend(db, &filter, tx, test_options)); + }); + + let mut status: MutantTestStatus = MutantTestStatus::Survived; + loop { + match rx.recv_timeout(config.mutate.test_timeout) { + Ok((_, suite_result)) => { + if suite_result.failures().count() > 0 { + status = MutantTestStatus::Killed; + break; + } + } + Err(std::sync::mpsc::RecvTimeoutError::Timeout) => { + trace!(target: "forge:mutate", "test timeout"); + status = MutantTestStatus::Timeout; + break; + } + _ => { + break; + } + } + } + + progress_bar.increment(); + + trace!(target: "forge::mutate", "received mutant test results"); + + (start.elapsed(), status) +} diff --git a/crates/forge/bin/cmd/mutate/summary.rs b/crates/forge/bin/cmd/mutate/summary.rs new file mode 100644 index 0000000000000..f44462e022978 --- /dev/null +++ b/crates/forge/bin/cmd/mutate/summary.rs @@ -0,0 +1,421 @@ +use comfy_table::{ + modifiers::UTF8_ROUND_CORNERS, Attribute, Cell, CellAlignment, Color, Row, Table, +}; +use core::fmt; +use eyre::{eyre, Result}; +use foundry_common::shell::{self}; +use foundry_evm_mutator::Mutant; +use serde::{ser::SerializeStruct, Deserialize, Serialize}; +use similar::TextDiff; +use std::{collections::BTreeMap, ops::Add, time::Duration}; +use yansi::Paint; + +const MAX_SURVIVE_RESULT_LOG_SIZE: usize = 5; + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub enum MutantTestStatus { + Killed, + Survived, + #[default] + Equivalent, + Timeout, +} + +impl fmt::Display for MutantTestStatus { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + MutantTestStatus::Killed => "KILLED".fmt(f), + MutantTestStatus::Survived => "SURVIVED".fmt(f), + MutantTestStatus::Equivalent => "EQUIVALENT".fmt(f), + MutantTestStatus::Timeout => "TIMEOUT".fmt(f), + } + } +} + +#[derive(Debug, Clone)] +pub struct MutantTestResult { + pub duration: Duration, + pub mutant: Mutant, + status: MutantTestStatus, +} + +impl MutantTestResult { + pub fn new(duration: Duration, mutant: Mutant, status: MutantTestStatus) -> Self { + Self { duration, mutant, status } + } + + pub fn killed(&self) -> bool { + matches!(self.status, MutantTestStatus::Killed) + } + + pub fn survived(&self) -> bool { + matches!(self.status, MutantTestStatus::Survived) + } + + pub fn equivalent(&self) -> bool { + matches!(self.status, MutantTestStatus::Equivalent) + } + + pub fn timeout(&self) -> bool { + matches!(self.status, MutantTestStatus::Timeout) + } + + pub fn diff(&self) -> String { + "".into() + } +} + +impl Serialize for MutantTestResult { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + let mut state = serializer.serialize_struct("MutantTestResult", 6)?; + state.serialize_field("name", self.mutant.source.filename())?; + state.serialize_field( + "path", + &self.mutant.source.sourceroot().join(&self.mutant.source.filename()).to_string_lossy(), + )?; + state.serialize_field("description", &self.mutant.op.to_string())?; + + let diff = mutant_diff(&self.mutant); + + state.serialize_field("diff", &diff)?; + state.serialize_field("result", &self.status.to_string())?; + + state.end() + } +} + +/// Results and duration for mutation tests for a contract +#[derive(Debug, Clone)] +pub struct MutationTestSuiteResult { + /// Total duration of the mutation tests run for this contract + pub duration: Duration, + /// Individual mutation test results. `file_name -> MutationTestResult` + mutation_test_results: Vec, +} + +impl MutationTestSuiteResult { + pub fn new(duration: Duration, results: Vec) -> Self { + Self { duration, mutation_test_results: results } + } + + pub fn killed(&self) -> impl Iterator { + self.mutation_test_results().filter(|result| result.killed()) + } + + pub fn survived(&self) -> impl Iterator { + self.mutation_test_results().filter(|result| result.survived()) + } + + pub fn equivalent(&self) -> impl Iterator { + self.mutation_test_results().filter(|result| result.equivalent()) + } + + pub fn timeout(&self) -> impl Iterator { + self.mutation_test_results().filter(|result| result.timeout()) + } + + pub fn mutation_test_results(&self) -> impl Iterator { + self.mutation_test_results.iter() + } + + pub fn is_empty(&self) -> bool { + self.mutation_test_results.is_empty() + } + + pub fn len(&self) -> usize { + self.mutation_test_results.len() + } +} + +/// Represents the bundled results of all tests +#[derive(Clone, Debug)] +pub struct MutationTestOutcome { + /// Whether failures are allowed + /// This enables to exit early + pub allow_failure: bool, + + // this would be Contract -> SuiteResult + pub test_suite_result: BTreeMap, +} + +impl MutationTestOutcome { + pub fn new( + allow_failure: bool, + test_suite_result: BTreeMap, + ) -> Self { + Self { allow_failure, test_suite_result } + } + + /// Total duration for tests + pub fn duration(&self) -> Duration { + self.test_suite_result + .values() + .map(|suite| suite.duration) + .fold(Duration::from_secs(0), |acc, duration| acc + duration) + } + + /// Iterator over all killed mutation tests + pub fn killed(&self) -> impl Iterator { + self.results().filter(|result| result.killed()) + } + + /// Iterator over all surviving mutation tests + pub fn survived(&self) -> impl Iterator { + self.results().filter(|result| result.survived()) + } + + /// Iterator over all equivalent mutation tests + pub fn equivalent(&self) -> impl Iterator { + self.results().filter(|result| result.equivalent()) + } + + /// Iterator over all timeout mutation tests + pub fn timeout(&self) -> impl Iterator { + self.results().filter(|result| result.timeout()) + } + + /// Iterator over all mutation tests and their names + pub fn results(&self) -> impl Iterator { + self.test_suite_result.values().flat_map(|suite| suite.mutation_test_results()) + } + + pub fn summary(&self) -> String { + let survived = self.survived().count(); + let result = if survived == 0 { Paint::green("ok") } else { Paint::red("FAILED") }; + format!( + "Mutation Test result: {}. {} killed; {} survived; {} equivalent; {} timeout, finished in {:.2?}", + result, + Paint::green(self.killed().count()), + Paint::red(survived), + Paint::yellow(self.equivalent().count()), + Paint::cyan(self.timeout().count()), + self.duration() + ) + } + + /// Checks if there is any surviving mutations and failures are disallowed + pub fn ensure_ok(&self) -> eyre::Result<()> { + let survived = self.survived().count(); + + if self.allow_failure || survived == 0 { + return Ok(()); + } + + if !shell::verbosity().is_normal() { + // skip printing and exit early + std::process::exit(1); + } + + println!(); + println!("Surviving Mutations:"); + + for (contract_name, suite_result) in self.test_suite_result.iter() { + let survived = suite_result.survived().count(); + if survived == 0 { + continue; + } + + let term = if survived > 1 { "mutations" } else { "mutation" }; + println!("Encountered {} surviving {term} in {}", survived, contract_name); + + for survive_result in suite_result.survived().take(MAX_SURVIVE_RESULT_LOG_SIZE) { + let description = survive_result.mutant.op.to_string(); + let (line, _) = survive_result + .mutant + .get_line_column() + .map_err(|x| eyre!(format!("{:?}", x)))?; + println!( + "\t Location: {}:{}, MutationType: {}", + survive_result.mutant.source.filename_as_str(), + line, + description + ); + } + + if survived > MAX_SURVIVE_RESULT_LOG_SIZE { + println!("\t More.."); + } + } + + println!(); + println!( + "Encountered a total of {} surviving mutations, {} mutations killed", + Paint::red(survived.to_string()), + Paint::green(self.killed().count().to_string()) + ); + std::process::exit(1); + } +} + +pub struct MutationTestSummaryReporter { + /// The mutation test summary table. + pub(crate) table: Table, + pub(crate) is_detailed: bool, +} + +impl MutationTestSummaryReporter { + pub(crate) fn new(is_detailed: bool) -> Self { + let mut table = Table::new(); + table.apply_modifier(UTF8_ROUND_CORNERS); + let mut row = Row::from(vec![ + Cell::new("File").set_alignment(CellAlignment::Left).add_attribute(Attribute::Bold), + Cell::new("Killed") + .set_alignment(CellAlignment::Center) + .add_attribute(Attribute::Bold) + .fg(Color::White), + Cell::new("Survived") + .set_alignment(CellAlignment::Center) + .add_attribute(Attribute::Bold) + .fg(Color::White), + Cell::new("Equivalent") + .set_alignment(CellAlignment::Center) + .add_attribute(Attribute::Bold) + .fg(Color::White), + Cell::new("Timeout") + .set_alignment(CellAlignment::Center) + .add_attribute(Attribute::Bold) + .fg(Color::White), + ]); + + if is_detailed { + row.add_cell( + Cell::new("Duration") + .set_alignment(CellAlignment::Center) + .add_attribute(Attribute::Bold), + ); + } + + row.add_cell( + Cell::new("% Score") + .set_alignment(CellAlignment::Center) + .add_attribute(Attribute::Bold) + .fg(Color::White), + ); + + table.set_header(row); + Self { table, is_detailed } + } + + pub fn print_summary(&mut self, mutation_test_outcome: &MutationTestOutcome) { + let mut total_killed = 0.0; + let mut total_survived = 0.0; + let mut total_equivalent = 0.0; + let mut total_timeout = 0.0; + let mut total_time_taken = Duration::ZERO; + for (contract_name, suite_result) in mutation_test_outcome.test_suite_result.iter() { + let mut row = Row::new(); + + let contract_title: String; + if let Some(result) = suite_result.mutation_test_results.first() { + contract_title = + format!("{}:{}", result.mutant.source.filename_as_str(), contract_name); + } else { + contract_title = contract_name.to_string(); + } + + let file_cell = Cell::new(contract_title).set_alignment(CellAlignment::Left); + row.add_cell(file_cell); + + let killed = suite_result.killed().count() as f64; + total_killed += killed; + let survived = suite_result.survived().count() as f64; + total_survived += survived; + let equivalent = suite_result.equivalent().count() as f64; + total_equivalent += equivalent; + let timeout = suite_result.timeout().count() as f64; + total_timeout += timeout; + + let mut killed_cell = Cell::new(killed).set_alignment(CellAlignment::Center); + let mut survived_cell = Cell::new(survived).set_alignment(CellAlignment::Center); + let mut equivalent_cell = Cell::new(equivalent).set_alignment(CellAlignment::Center); + let mut timeout_cell = Cell::new(timeout).set_alignment(CellAlignment::Center); + + if killed > 0.0 { + killed_cell = killed_cell.fg(Color::Green); + } + row.add_cell(killed_cell); + + if survived > 0.0 { + survived_cell = survived_cell.fg(Color::Red); + } + row.add_cell(survived_cell); + + if equivalent > 0.0 { + equivalent_cell = equivalent_cell.fg(Color::Yellow); + } + row.add_cell(equivalent_cell); + + if timeout > 0.0 { + timeout_cell = timeout_cell.fg(Color::Cyan); + } + row.add_cell(timeout_cell); + + if self.is_detailed { + total_time_taken = total_time_taken.add(suite_result.duration); + row.add_cell(Cell::new(format!("{:.2?}", suite_result.duration).to_string())); + } + + let mut mutation_score: f64 = 0.0; + if killed > 0.0 { + mutation_score = ((killed / (killed + survived)) * 100.0) as f64; + } + let mut mutation_score_cell = Cell::new(format!("{:.2}", mutation_score).to_string()) + .set_alignment(CellAlignment::Center); + + mutation_score_cell = if mutation_score > 50.0 { + mutation_score_cell.fg(Color::Green) + } else { + mutation_score_cell.fg(Color::Red) + }; + + row.add_cell(mutation_score_cell); + + self.table.add_row(row); + } + + let mut footer = Row::from(vec![ + Cell::new("Total").set_alignment(CellAlignment::Center), + Cell::new(total_killed).set_alignment(CellAlignment::Center), + Cell::new(total_survived).set_alignment(CellAlignment::Center), + Cell::new(total_equivalent).set_alignment(CellAlignment::Center), + Cell::new(total_timeout).set_alignment(CellAlignment::Center), + ]); + + if self.is_detailed { + footer.add_cell( + Cell::new(format!("{:.2?}", total_time_taken).to_string()) + .set_alignment(CellAlignment::Left), + ); + } + + let mut mutation_score: f64 = 0.0; + if total_killed > 0.0 { + mutation_score = ((total_killed / (total_killed + total_survived)) * 100.0) as f64; + } + let mut mutation_score_cell = Cell::new(format!("{:.2}", mutation_score).to_string()) + .set_alignment(CellAlignment::Center); + mutation_score_cell = if mutation_score > 50.0 { + mutation_score_cell.fg(Color::Green) + } else { + mutation_score_cell.fg(Color::Red) + }; + + footer.add_cell(mutation_score_cell); + self.table.add_row(footer); + + println!("\n{}", self.table); + } +} + +pub fn mutant_diff(mutant: &Mutant) -> String { + let orig_contents: String = String::from_utf8_lossy(mutant.source.contents()).into(); + let mutant_contents = mutant.as_source_string().unwrap(); + + TextDiff::from_lines(&orig_contents, &mutant_contents) + .unified_diff() + .header("original", "mutant") + .to_string() +} diff --git a/crates/forge/bin/cmd/test/filter.rs b/crates/forge/bin/cmd/test/filter.rs index ec2e9b01b50e8..7f94c2e463a6c 100644 --- a/crates/forge/bin/cmd/test/filter.rs +++ b/crates/forge/bin/cmd/test/filter.rs @@ -80,6 +80,37 @@ impl FilterArgs { } ProjectPathsAwareFilter { args_filter: self, paths: config.project_paths() } } + + /// Merges the set filter globs with mutate test config values + pub fn merge_with_mutate_config(&self, config: &Config) -> ProjectPathsAwareFilter { + let mut filter = self.clone(); + + if filter.test_pattern.is_none() { + filter.test_pattern = config.mutate.test_pattern.clone().map(|p| p.into()); + } + + if filter.test_pattern_inverse.is_none() { + filter.test_pattern_inverse = config.mutate.test_contract_pattern_inverse.clone().map(|p| p.into()); + } + + if filter.contract_pattern.is_none() { + filter.contract_pattern = config.mutate.test_contract_pattern.clone().map(|p| p.into()); + } + + if filter.contract_pattern_inverse.is_none() { + filter.contract_pattern = config.mutate.test_contract_pattern_inverse.clone().map(|p| p.into()); + } + + if filter.path_pattern.is_none() { + filter.path_pattern = config.mutate.test_path_pattern.clone().map(|p| p.into()); + } + + if filter.path_pattern_inverse.is_none() { + filter.path_pattern = config.mutate.test_path_pattern_inverse.clone().map(|p| p.into()); + } + + ProjectPathsAwareFilter { args_filter: self.clone(), paths: config.project_paths() } + } } impl fmt::Debug for FilterArgs { diff --git a/crates/forge/bin/cmd/test/mod.rs b/crates/forge/bin/cmd/test/mod.rs index ab868f5bc9d06..d1f8a031d155f 100644 --- a/crates/forge/bin/cmd/test/mod.rs +++ b/crates/forge/bin/cmd/test/mod.rs @@ -1,4 +1,4 @@ -use super::{install, test::filter::ProjectPathsAwareFilter, watch::WatchArgs}; +use super::{install, watch::WatchArgs}; use alloy_primitives::U256; use clap::Parser; use eyre::Result; @@ -49,7 +49,7 @@ mod filter; mod summary; use summary::TestSummaryReporter; -pub use filter::FilterArgs; +pub use filter::{FilterArgs, ProjectPathsAwareFilter}; // Loads project's figment and merges the build cli arguments into it foundry_config::merge_impl_figment_convert!(TestArgs, opts, evm_opts); diff --git a/crates/forge/bin/main.rs b/crates/forge/bin/main.rs index dc4a18dd6840a..0680dc6e1f746 100644 --- a/crates/forge/bin/main.rs +++ b/crates/forge/bin/main.rs @@ -121,6 +121,10 @@ fn main() -> Result<()> { ForgeSubcommand::Soldeer(cmd) => cmd.run(), ForgeSubcommand::Eip712(cmd) => cmd.run(), ForgeSubcommand::BindJson(cmd) => cmd.run(), + ForgeSubcommands::Mutate(cmd) => { + utils::block_on(cmd.run()) + } + } } diff --git a/crates/forge/bin/opts.rs b/crates/forge/bin/opts.rs index b86d19c17728d..0bf6283f08b21 100644 --- a/crates/forge/bin/opts.rs +++ b/crates/forge/bin/opts.rs @@ -3,6 +3,7 @@ use crate::cmd::{ coverage, create::CreateArgs, debug::DebugArgs, doc::DocArgs, eip712, flatten, fmt::FmtArgs, geiger, generate, init::InitArgs, inspect, install::InstallArgs, remappings::RemappingArgs, remove::RemoveArgs, selectors::SelectorsSubcommands, snapshot, soldeer, test, tree, update, + mutate::MutateTestArgs, }; use clap::{Parser, Subcommand, ValueHint}; use forge_script::ScriptArgs; @@ -157,7 +158,7 @@ pub enum ForgeSubcommand { /// Generate scaffold files. Generate(generate::GenerateArgs), - + /// Verify the deployed bytecode against its source. #[clap(visible_alias = "vb")] VerifyBytecode(VerifyBytecodeArgs), @@ -170,6 +171,9 @@ pub enum ForgeSubcommand { /// Generate bindings for serialization/deserialization of project structs via JSON cheatcodes. BindJson(bind_json::BindJsonArgs), + + /// Run mutation tests on project + Mutate(MutateTestArgs), } #[cfg(test)] diff --git a/crates/forge/src/multi_runner.rs b/crates/forge/src/multi_runner.rs index 4df00f7944eee..42231f72c2ca5 100644 --- a/crates/forge/src/multi_runner.rs +++ b/crates/forge/src/multi_runner.rs @@ -183,7 +183,7 @@ impl MultiContractRunner { self.contracts.len(), find_time, ); - + if show_progress { let tests_progress = TestsProgress::new(contracts.len(), rayon::current_num_threads()); // Collect test suite results to stream at the end of test run. diff --git a/crates/forge/src/runner.rs b/crates/forge/src/runner.rs index 6ed06e525e399..a384e53a42cd6 100644 --- a/crates/forge/src/runner.rs +++ b/crates/forge/src/runner.rs @@ -407,7 +407,7 @@ impl<'a> ContractRunner<'a> { (sig, res) }) .collect::>(); - + let duration = start.elapsed(); SuiteResult::new(duration, test_results, warnings) } @@ -642,14 +642,12 @@ impl<'a> ContractRunner<'a> { fuzz_config: FuzzConfig, ) -> TestResult { let progress = start_fuzz_progress(self.progress, self.name, &func.name, fuzz_config.runs); - // Prepare fuzz test execution. let fuzz_fixtures = setup.fuzz_fixtures.clone(); let (executor, test_result, address) = match self.prepare_test(func, setup) { Ok(res) => res, Err(res) => return res, }; - // Run fuzz test. let fuzzed_executor = FuzzedExecutor::new(executor.into_owned(), runner, self.sender, fuzz_config); diff --git a/crates/forge/tests/cli/config.rs b/crates/forge/tests/cli/config.rs index 0e38e15e7d87b..e36d1156f1ee7 100644 --- a/crates/forge/tests/cli/config.rs +++ b/crates/forge/tests/cli/config.rs @@ -149,6 +149,7 @@ forgetest!(can_extract_config_values, |prj, cmd| { extra_args: vec![], eof_version: None, alphanet: false, + mutate: Default::default(), _non_exhaustive: (), }; prj.write_config(input.clone());