Intruduce an E2E testing framework

Add a tests directory containing an E2E testing framework.
The tests are E2E in the sense that they use the public API of libkrun to
start a VM and run a test program in the guest.
Note that currently this is very limited, because there are no other userspace
executables or libraries (not even libc) in the guest apart from `guest-agent`.

`guest-agent` is a statically linked Rust executable, that executes the guest
part of each test.

In the future this can be extended to run tests with specific images which would
allow the use of system libraries by the test program.

This also introduces 2 tests:
test_vm_config - asserts the VM is constructed with the correct number of CPUs
                 and amount of memory. This also serves as an example how the
		 tests can be parameterized.
test_vsock_guest_connect - This tests the guest connecting to a vsock port
                           created by the `krun_add_vsock_port` API.

Signed-off-by: Matej Hrica <[email protected]>

Author: mtjhrc

tests/Cargo.lock

@@ -0,0 +1,3 @@
+[workspace]
+members = ["runner", "guest-agent", "macros", "test_cases"]
+resolver = "2"

tests/Cargo.toml

@@ -0,0 +1,9 @@
+# End-to-end tests
+The testing framework here allows you to write code to configure libkrun (using the public API) and run some specific code in the guest. 
+
+## Running the tests:
+The tests can be ran using `make test` (from the main libkrun directory).
+You can also run `./run.sh` inside the `test` directory. When using the `./run.sh` script you probably want specify the `PKG_CONFIG_PATH` enviroment variable, otherwise you will be testing the system wide installation of libkrun. 
+
+## Adding tests
+To add a test you need to add a new rust module in the `test_cases` directory, implement the  required host and guest side methods (see existing tests) and register the test in the `test_cases/src/lib.rs` to be ran.

tests/README.md

@@ -0,0 +1,7 @@
+[package]
+name = "guest-agent"
+edition = "2021"
+
+[dependencies]
+test_cases = { path = "../test_cases", features = ["guest"] }
+anyhow = "1.0.95"

tests/guest-agent/Cargo.toml

@@ -0,0 +1,22 @@
+use anyhow::Context;
+use std::env::args;
+use test_cases::{test_cases, TestCase};
+
+fn run_guest_agent(test_name: &str) -> anyhow::Result<()> {
+    let tests = test_cases();
+    let test_case = tests
+        .into_iter()
+        .find(|t| t.name() == test_name)
+        .context("No such test!")?;
+    let TestCase { test, name: _ } = test_case;
+    test.in_guest();
+    Ok(())
+}
+
+fn main() -> anyhow::Result<()> {
+    let mut cli_args = args();
+    let _exec_name = cli_args.next();
+    let test_name = cli_args.next().context("Missing test name argument")?;
+    run_guest_agent(&test_name)?;
+    Ok(())
+}

tests/guest-agent/src/main.rs

@@ -0,0 +1,10 @@
+[package]
+name = "macros"
+edition = "2021"
+
+[lib]
+proc-macro = true
+
+[dependencies]
+syn = "2.0.96"
+quote = "1.0.38"

tests/macros/Cargo.toml

@@ -0,0 +1,28 @@
+extern crate proc_macro;
+extern crate quote;
+extern crate syn;
+
+use proc_macro::TokenStream;
+use quote::quote;
+
+#[proc_macro_attribute]
+pub fn guest(_args: TokenStream, input: TokenStream) -> TokenStream {
+    let mut prefix: TokenStream = quote! {
+        #[cfg(feature = "guest")]
+    }
+    .into();
+
+    prefix.extend(input);
+    prefix.into()
+}
+
+#[proc_macro_attribute]
+pub fn host(_args: TokenStream, input: TokenStream) -> TokenStream {
+    let mut prefix: TokenStream = quote! {
+         #[cfg(feature = "host")]
+    }
+    .into();
+
+    prefix.extend(input);
+    prefix.into()
+}

tests/macros/src/lib.rs

@@ -0,0 +1,15 @@
+#/bin/sh
+
+# This script has to be run with the working directory being "test"
+# This runs the tests on the libkrun instance found by pkg-config.
+# Specify PKG_CONFIG_PATH env variable to test a non-system installation of libkurn.
+
+set -e
+
+# Run the unit tests first (this tests the testing framework itself not libkrun)
+cargo test -p test_cases --features guest
+
+GUEST_TARGET_ARCH="$(uname -m)-unknown-linux-musl"
+
+cargo build --target=$GUEST_TARGET_ARCH -p guest-agent
+KRUN_TEST_GUEST_AGENT_PATH="target/$GUEST_TARGET_ARCH/debug/guest-agent" cargo run -p runner "$@"

tests/run.sh

@@ -0,0 +1,11 @@
+[package]
+name = "runner"
+edition = "2021"
+
+[dependencies]
+test_cases = { path = "../test_cases", features = ["host"] }
+anyhow = "1.0.95"
+nix = { version = "0.29.0", features = ["resource"] }
+macros = { path = "../macros" }
+clap = { version = "4.5.27", features = ["derive"] }
+tempdir = "0.3.7"

tests/runner/Cargo.toml

@@ -0,0 +1,132 @@
+use anyhow::Context;
+use clap::Parser;
+use nix::sys::resource::{getrlimit, setrlimit, Resource};
+use std::panic::catch_unwind;
+use std::path::PathBuf;
+use std::process::{Command, Stdio};
+use std::{env, mem};
+use tempdir::TempDir;
+use test_cases::{test_cases, Test, TestCase, TestSetup};
+
+fn get_test(name: &str) -> anyhow::Result<Box<dyn Test>> {
+    let tests = test_cases();
+    tests
+        .into_iter()
+        .find(|t| t.name() == name)
+        .with_context(|| format!("No such test: {name}"))
+        .map(|t| t.test)
+}
+
+fn start_vm(test_setup: TestSetup) -> anyhow::Result<()> {
+    // Raise soft fd limit up to the hard limit
+    let (_soft_limit, hard_limit) =
+        getrlimit(Resource::RLIMIT_NOFILE).context("getrlimit RLIMIT_NOFILE")?;
+    setrlimit(Resource::RLIMIT_NOFILE, hard_limit, hard_limit)
+        .context("setrlimit RLIMIT_NOFILE")?;
+
+    let test = get_test(&test_setup.test_case)?;
+    test.start_vm(test_setup.clone())
+        .with_context(|| format!("testcase: {test_setup:?}"))?;
+    Ok(())
+}
+
+fn run_single_test(test_case: &str) -> anyhow::Result<bool> {
+    let executable = env::current_exe().context("Failed to detect current executable")?;
+    let tmp_dir =
+        TempDir::new(&format!("krun-test-{test_case}")).context("Failed to create tmp dir")?;
+
+    let child = Command::new(&executable)
+        .arg("start-vm")
+        .arg("--test-case")
+        .arg(test_case)
+        .arg("--tmp-dir")
+        .arg(tmp_dir.path())
+        .stdin(Stdio::piped())
+        .stdout(Stdio::piped())
+        .stderr(Stdio::piped())
+        .spawn()
+        .context("Failed to start subprocess for test")?;
+
+    let _ = get_test(test_case)?;
+    let result = catch_unwind(|| {
+        let test = get_test(test_case).unwrap();
+        test.check(child);
+    });
+
+    match result {
+        Ok(()) => {
+            println!("[{test_case}]: OK");
+            Ok(true)
+        }
+        Err(_e) => {
+            println!("[{test_case}]: FAIL (dir {:?} kept)", tmp_dir.path());
+            mem::forget(tmp_dir);
+            Ok(false)
+        }
+    }
+}
+
+fn run_tests(test_case: &str) -> anyhow::Result<()> {
+    let mut num_tests = 1;
+    let mut num_ok: usize = 0;
+
+    if test_case == "all" {
+        let test_cases = test_cases();
+        num_tests = test_cases.len();
+
+        for TestCase { name, test: _ } in test_cases {
+            num_ok += run_single_test(name).context(name)? as usize;
+        }
+    } else {
+        num_ok += run_single_test(test_case).context(test_case.to_string())? as usize;
+    }
+
+    let num_failures = num_tests - num_ok;
+    if num_failures > 0 {
+        println!("\nFAIL (PASSED {num_ok}/{num_tests})");
+        anyhow::bail!("")
+    } else {
+        println!("\nOK (PASSED {num_ok}/{num_tests})");
+    }
+
+    Ok(())
+}
+
+#[derive(clap::Subcommand, Clone, Debug)]
+enum CliCommand {
+    Test {
+        /// Specify which test to run or "all"
+        #[arg(long, default_value = "all")]
+        test_case: String,
+    },
+    StartVm {
+        #[arg(long)]
+        test_case: String,
+        #[arg(long)]
+        tmp_dir: PathBuf,
+    },
+}
+
+impl Default for CliCommand {
+    fn default() -> Self {
+        Self::Test {
+            test_case: "all".to_string(),
+        }
+    }
+}
+
+#[derive(clap::Parser)]
+struct Cli {
+    #[command(subcommand)]
+    command: Option<CliCommand>,
+}
+
+fn main() -> anyhow::Result<()> {
+    let cli = Cli::parse();
+    let command = cli.command.unwrap_or_default();
+
+    match command {
+        CliCommand::StartVm { test_case, tmp_dir } => start_vm(TestSetup { test_case, tmp_dir }),
+        CliCommand::Test { test_case } => run_tests(&test_case),
+    }
+}

tests/runner/src/main.rs

@@ -0,0 +1,17 @@
+[package]
+name = "test_cases"
+edition = "2021"
+
+[features]
+host = ["krun-sys"]
+guest = []
+
+[lib]
+name = "test_cases"
+
+[dependencies]
+krun-sys = { path = "../../krun-sys", optional = true }
+macros = { path = "../macros" }
+nix = { version = "0.29.0", features = ["socket"] }
+anyhow = "1.0.95"
+tempdir = "0.3.7"

tests/test_cases/Cargo.toml

@@ -0,0 +1,52 @@
+//! Common utilities used by multiple test
+
+use anyhow::Context;
+use std::ffi::CString;
+use std::fs;
+use std::fs::create_dir;
+use std::os::unix::ffi::OsStrExt;
+use std::path::Path;
+use std::ptr::null;
+
+use crate::{krun_call, TestSetup};
+use krun_sys::*;
+
+fn copy_guest_agent(dir: &Path) -> anyhow::Result<()> {
+    let path = std::env::var_os("KRUN_TEST_GUEST_AGENT_PATH")
+        .context("KRUN_TEST_GUEST_AGENT_PATH env variable not set")?;
+
+    let output_path = dir.join("guest-agent");
+    fs::copy(path, output_path).context("Failed to copy executable into vm")?;
+    Ok(())
+}
+
+/// Common part of most test. This setups an empty root filesystem, copies the guest agent there
+/// and runs the guest agent in the VM.
+/// Note that some tests might want to use a different root file system (perhaps a qcow image),
+/// in which case the test can implement the equivalent functionality itself, or better if there
+/// are more test doing that, add another utility method in this file.
+///
+/// The returned object is used for deleting the temporary files.
+pub fn setup_fs_and_enter(ctx: u32, test_setup: TestSetup) -> anyhow::Result<()> {
+    let root_dir = test_setup.tmp_dir.join("root");
+    create_dir(&root_dir).context("Failed to create root directory")?;
+
+    let path_str = CString::new(root_dir.as_os_str().as_bytes()).context("CString::new")?;
+    copy_guest_agent(&root_dir)?;
+    unsafe {
+        krun_call!(krun_set_root(ctx, path_str.as_ptr()))?;
+        krun_call!(krun_set_workdir(ctx, c"/".as_ptr()))?;
+        let test_case_cstr = CString::new(test_setup.test_case).context("CString::new")?;
+        let argv = [test_case_cstr.as_ptr(), null()];
+        //let envp = [c"RUST_BACKTRACE=1".as_ptr(), null()];
+        let envp = [null()];
+        krun_call!(krun_set_exec(
+            ctx,
+            c"/guest-agent".as_ptr(),
+            argv.as_ptr(),
+            envp.as_ptr(),
+        ))?;
+        krun_call!(krun_start_enter(ctx))?;
+    }
+    unreachable!()
+}

tests/test_cases/src/common.rs

@@ -0,0 +1,25 @@
+#[macro_export]
+macro_rules! krun_call {
+    ($func_call:expr) => {{
+        let result = $func_call;
+        if result < 0 {
+            let err = std::io::Error::from_raw_os_error(-result);
+            Err(anyhow::anyhow!("`{}`: {}", stringify!($func_call), err))
+        } else {
+            Ok::<(), anyhow::Error>(())
+        }
+    }};
+}
+
+#[macro_export]
+macro_rules! krun_call_u32 {
+    ($func_call:expr) => {{
+        let result = $func_call;
+        if result < 0 {
+            let err = std::io::Error::from_raw_os_error(-result);
+            Err(anyhow::anyhow!("`{}`: {}", stringify!($func_call), err))
+        } else {
+            Ok::<u32, anyhow::Error>(result as u32)
+        }
+    }};
+}