tasks: add tests task

Author: nyonson

tasks/README.md

@@ -4,11 +4,11 @@ Maintainer tools for Rust-based projects in the Bitcoin domain. Built with [xshe
 
 ## Configuration
 
-Configuration for `rbmt` is stored in `contrib/rbmt.toml`.
+Configuration for `rbmt` is stored in `contrib/rbmt.toml`. The file can live at both the workspace root (e.g. `$ROOT/contrib/rbmt.toml`) as well as per-crate (e.g. `$ROOT/$CRATE/contrib/rbmt.toml`) within a repository.
 
 ### Lint
 
-The `lint` command detects duplicate dependencies, but some may be unavoidable (e.g., during dependency updates where transitive dependencies haven't caught up). Configure the `[lint]` section to whitelist specific duplicates.
+The `lint` command detects duplicate dependencies, but some may be unavoidable (e.g., during dependency updates where transitive dependencies haven't caught up). Configure the `[lint]` section to whitelist specific duplicates for a workspace (or a crate if only one crate in a repository).
 
 ```toml
 [lint]
@@ -18,6 +18,45 @@ allowed_duplicates = [
 ]
 ```
 
+### Test
+
+The `test` command can be configured to run feature matrix testing for your crate. Configure with the `contrib/rbmt.toml` file at the crate level.
+
+```toml
+[test]
+# Examples to run with specific features enabled.
+# Format: "example_name:feature1 feature2"
+examples = [
+    "example1:serde",
+    "example2:serde rand",
+]
+
+# Features to test with the conventional `std` feature enabled.
+# Tests each feature alone with std, all pairs, and all together.
+# Example: ["serde", "rand"] tests: std+serde, std+rand, std+serde+rand
+features_with_std = ["serde", "rand"]
+
+# Features to test without the `std` feature.
+# Tests each feature alone, all pairs, and all together.
+# Example: ["serde", "rand"] tests: serde, rand, serde+rand
+features_without_std = ["serde", "rand"]
+
+# Exact feature combinations to test.
+# Use for crates that don't follow conventional `std` patterns.
+# Each inner array is tested as-is with no automatic combinations.
+# Example: [["serde", "rand"], ["rand"]] tests exactly those two combinations
+exact_features = [
+    ["serde", "rand"],
+    ["rand"],
+]
+
+# Features to test with an explicit `no-std` feature enabled.
+# Only use if your crate has a `no-std` feature (rust-miniscript pattern).
+# Tests each feature with no-std, all pairs, and all together.
+# Example: ["serde", "rand"] tests: no-std+serde, no-std+rand, no-std+serde+rand
+features_with_no_std = ["serde", "rand"]
+```
+
 ### Environment Variables
 
 * `RBMT_LOG_LEVEL=quiet` - Suppress verbose output and reduce cargo noise.

tasks/src/main.rs

@@ -2,13 +2,15 @@ mod bench;
 mod docs;
 mod environment;
 mod lint;
+mod test;
 mod toolchain;
 
 use clap::{Parser, Subcommand};
 use std::process;
 use xshell::Shell;
 
 use environment::{change_to_repo_root, configure_log_level};
+use toolchain::Toolchain;
 
 #[derive(Parser)]
 #[command(name = "rbmt")]
@@ -28,6 +30,12 @@ enum Commands {
     Docsrs,
     /// Run benchmark tests for all crates.
     Bench,
+    /// Run tests with specified toolchain.
+    Test {
+        /// Toolchain to use: stable, nightly, or msrv.
+        #[arg(value_enum)]
+        toolchain: Toolchain,
+    },
 }
 
 fn main() {
@@ -61,5 +69,11 @@ fn main() {
                 process::exit(1);
             }
         }
+        Commands::Test { toolchain } => {
+            if let Err(e) = test::run(&sh, toolchain) {
+                eprintln!("Error running tests: {}", e);
+                process::exit(1);
+            }
+        }
     }
 }

tasks/src/test.rs

@@ -0,0 +1,260 @@
+//! Build and test tasks with feature matrix testing.
+
+use crate::environment::{get_crate_dirs, quiet_println, CONFIG_FILE_PATH};
+use crate::quiet_cmd;
+use crate::toolchain::{check_toolchain, Toolchain};
+use serde::Deserialize;
+use std::path::Path;
+use xshell::Shell;
+
+/// Test configuration loaded from contrib/rbmt.toml.
+#[derive(Debug, Deserialize)]
+struct Config {
+    test: TestConfig,
+}
+
+/// Test-specific configuration.
+#[derive(Debug, Deserialize)]
+struct TestConfig {
+    /// Examples to run with the format "name:feature1 feature2".
+    ///
+    /// # Examples
+    ///
+    /// `["example1:serde", "example2:serde rand"]`
+    examples: Vec<String>,
+
+    /// List of individual features to test with the conventional `std` feature enabled.
+    /// Automatically tests feature combinations, alone with `std`, all pairs, and all together.
+    ///
+    /// # Examples
+    ///
+    /// `["serde", "rand"]` tests `std+serde`, `std+rand`, `std+serde+rand`.
+    features_with_std: Vec<String>,
+
+    /// List of individual features to test without the `std` feature.
+    /// Automatically tests features combinations, each feature alone,
+    /// all pairs, and all together.
+    ///
+    /// # Examples
+    ///
+    /// `["serde", "rand"]` tests `serde`, `rand`, `serde+rand`.
+    features_without_std: Vec<String>,
+
+    /// Exact feature combinations to test.
+    /// Use for crates that don't follow the conventional `std` feature pattern.
+    /// Each inner vector is a list of features to test together. There is
+    /// no automatic combinations of features tests.
+    ///
+    /// # Examples
+    ///
+    /// `[["serde", "rand"], ["rand"]]` tests exactly those two combinations.
+    exact_features: Vec<Vec<String>>,
+
+    /// List of individual features to test with the `no-std` feature enabled.
+    /// Only use if your crate has an explicit `no-std` feature (rust-miniscript pattern).
+    /// Automatically tests each feature alone with `no-std`, all pairs, and all together.
+    ///
+    /// # Examples
+    ///
+    /// `["serde", "rand"]` tests `no-std+serde`, `no-std+serde`, `no-std+serde+rand`.
+    features_with_no_std: Vec<String>,
+}
+
+impl TestConfig {
+    /// Load test configuration from a crate directory.
+    fn load(crate_dir: &Path) -> Result<Self, Box<dyn std::error::Error>> {
+        let config_path = crate_dir.join(CONFIG_FILE_PATH);
+
+        if !config_path.exists() {
+            // Return empty config if file doesn't exist.
+            return Ok(TestConfig {
+                examples: Vec::new(),
+                features_with_std: Vec::new(),
+                features_without_std: Vec::new(),
+                exact_features: Vec::new(),
+                features_with_no_std: Vec::new(),
+            });
+        }
+
+        let contents = std::fs::read_to_string(&config_path)?;
+        let config: Config = toml::from_str(&contents)?;
+        Ok(config.test)
+    }
+}
+
+/// Run build and test for all crates with the specified toolchain.
+pub fn run(sh: &Shell, toolchain: Toolchain) -> Result<(), Box<dyn std::error::Error>> {
+    check_toolchain(sh, toolchain)?;
+
+    let crate_dirs = get_crate_dirs(sh)?;
+    quiet_println(&format!("Testing {} crates", crate_dirs.len()));
+
+    for crate_dir in &crate_dirs {
+        quiet_println(&format!("Testing crate: {}", crate_dir));
+
+        let _dir = sh.push_dir(crate_dir);
+        let config = TestConfig::load(Path::new(crate_dir))?;
+
+        do_test(sh, &config)?;
+        do_feature_matrix(sh, &config)?;
+    }
+
+    Ok(())
+}
+
+/// Run basic build, test, and examples.
+fn do_test(sh: &Shell, config: &TestConfig) -> Result<(), Box<dyn std::error::Error>> {
+    quiet_println("Running basic tests");
+
+    // Basic build and test.
+    quiet_cmd!(sh, "cargo build").run()?;
+    quiet_cmd!(sh, "cargo test").run()?;
+
+    // Run examples.
+    for example in &config.examples {
+        let parts: Vec<&str> = example.split(':').collect();
+        if parts.len() != 2 {
+            return Err(format!(
+                "Invalid example format: {}, expected 'name:features'",
+                example
+            )
+            .into());
+        }
+
+        let name = parts[0];
+        let features = parts[1];
+
+        quiet_println(&format!(
+            "Running example {} with features: {}",
+            name, features
+        ));
+        quiet_cmd!(sh, "cargo run --example {name} --features={features}").run()?;
+    }
+
+    Ok(())
+}
+
+/// Run feature matrix tests.
+fn do_feature_matrix(sh: &Shell, config: &TestConfig) -> Result<(), Box<dyn std::error::Error>> {
+    quiet_println("Running feature matrix tests");
+
+    // Handle exact features (for unusual crates).
+    if !config.exact_features.is_empty() {
+        for features in &config.exact_features {
+            let features_str = features.join(" ");
+            quiet_println(&format!("Testing exact features: {}", features_str));
+            quiet_cmd!(
+                sh,
+                "cargo build --no-default-features --features={features_str}"
+            )
+            .run()?;
+            quiet_cmd!(
+                sh,
+                "cargo test --no-default-features --features={features_str}"
+            )
+            .run()?;
+        }
+        return Ok(());
+    }
+
+    // Handle no-std pattern (rust-miniscript).
+    if !config.features_with_no_std.is_empty() {
+        quiet_println("Testing no-std");
+        quiet_cmd!(sh, "cargo build --no-default-features --features=no-std").run()?;
+        quiet_cmd!(sh, "cargo test --no-default-features --features=no-std").run()?;
+
+        loop_features(sh, "no-std", &config.features_with_no_std)?;
+    } else {
+        quiet_println("Testing no-default-features");
+        quiet_cmd!(sh, "cargo build --no-default-features").run()?;
+        quiet_cmd!(sh, "cargo test --no-default-features").run()?;
+    }
+
+    // Test all features.
+    quiet_println("Testing all-features");
+    quiet_cmd!(sh, "cargo build --all-features").run()?;
+    quiet_cmd!(sh, "cargo test --all-features").run()?;
+
+    // Test features with std.
+    if !config.features_with_std.is_empty() {
+        loop_features(sh, "std", &config.features_with_std)?;
+    }
+
+    // Test features without std.
+    if !config.features_without_std.is_empty() {
+        loop_features(sh, "", &config.features_without_std)?;
+    }
+
+    Ok(())
+}
+
+/// Test each feature individually and all combinations of two features.
+///
+/// This implements three feature matrix testing strategies.
+/// 1. All features together.
+/// 2. Each feature individually (only if more than one feature).
+/// 3. All unique pairs of features.
+///
+/// The pair testing catches feature interaction bugs (where two features work
+/// independently, but conflict when combined) while keeping test time manageable.
+fn loop_features(
+    sh: &Shell,
+    base: &str,
+    features: &[String],
+) -> Result<(), Box<dyn std::error::Error>> {
+    let base_flag = if base.is_empty() {
+        String::new()
+    } else {
+        format!("{} ", base)
+    };
+
+    // Test all features together.
+    let all_features = format!("{}{}", base_flag, features.join(" "));
+    quiet_println(&format!("Testing features: {}", all_features.trim()));
+    quiet_cmd!(
+        sh,
+        "cargo build --no-default-features --features={all_features}"
+    )
+    .run()?;
+    quiet_cmd!(
+        sh,
+        "cargo test --no-default-features --features={all_features}"
+    )
+    .run()?;
+
+    // Test each feature individually and all pairs (only if more than one feature).
+    if features.len() > 1 {
+        for i in 0..features.len() {
+            let feature_combo = format!("{}{}", base_flag, features[i]);
+            quiet_println(&format!("Testing features: {}", feature_combo.trim()));
+            quiet_cmd!(
+                sh,
+                "cargo build --no-default-features --features={feature_combo}"
+            )
+            .run()?;
+            quiet_cmd!(
+                sh,
+                "cargo test --no-default-features --features={feature_combo}"
+            )
+            .run()?;
+
+            // Test all pairs with features[i].
+            for j in (i + 1)..features.len() {
+                let feature_combo = format!("{}{} {}", base_flag, features[i], features[j]);
+                quiet_println(&format!("Testing features: {}", feature_combo.trim()));
+                quiet_cmd!(
+                    sh,
+                    "cargo build --no-default-features --features={feature_combo}"
+                )
+                .run()?;
+                quiet_cmd!(
+                    sh,
+                    "cargo test --no-default-features --features={feature_combo}"
+                )
+                .run()?;
+            }
+        }
+    }
+
+    Ok(())
+}

tasks/src/toolchain.rs

@@ -2,7 +2,7 @@ use std::path::Path;
 use xshell::{cmd, Shell};
 
 /// Toolchain requirement for a task.
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum)]
 pub enum Toolchain {
     /// Nightly toolchain.
     Nightly,
@@ -92,7 +92,10 @@ fn get_msrv_from_manifest(manifest_path: &Path) -> Result<String, Box<dyn std::e
 }
 
 /// Extract version number from rustc --version output.
-/// Example: "rustc 1.74.0 (79e9716c9 2023-11-13)" -> Some("1.74.0")
+///
+/// # Examples
+///
+/// `"rustc 1.74.0 (79e9716c9 2023-11-13)"` -> `Some("1.74.0")`
 fn extract_version(rustc_version: &str) -> Option<&str> {
     rustc_version.split_whitespace().find_map(|part| {
         // Split off any suffix like "-nightly" or "-beta".