diff --git a/.github/workflows/rust-ci.yaml b/.github/workflows/rust-ci.yaml index 9981c572..02a7e15b 100644 --- a/.github/workflows/rust-ci.yaml +++ b/.github/workflows/rust-ci.yaml @@ -107,7 +107,16 @@ jobs: matrix: platform: [ public-ubuntu-24.04-32core, macos-14 ] runs-on: ${{ matrix.platform }} + permissions: + id-token: write + contents: read steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: eu-central-1 + role-to-assume: ${{ secrets.AWS_ROLE }} + role-duration-seconds: 7200 - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # pin@v3 with: token: ${{ secrets.ORB_GIT_HUB_TOKEN }} diff --git a/Cargo.lock b/Cargo.lock index 248697a5..b815ab93 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3674,6 +3674,16 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs4" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8640e34b88f7652208ce9e88b1a37a2ae95227d84abec377ccd3c5cfeb141ed4" +dependencies = [ + "rustix 1.0.8", + "windows-sys 0.59.0", +] + [[package]] name = "ftdi-embedded-hal" version = "0.22.0" @@ -9768,9 +9778,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.11.1" +version = "1.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +checksum = "23d7fd106d8c02486a8d64e778353d1cffe08ce79ac2e82f540c86d0facf6912" dependencies = [ "aho-corasick", "memchr", @@ -11803,10 +11813,15 @@ dependencies = [ name = "test-utils" version = "0.1.0" dependencies = [ + "blake3", + "cmd_lib", + "fs4", "nix 0.28.0", + "regex", "tempfile", "testcontainers", "tokio", + "uuid", ] [[package]] @@ -12822,9 +12837,9 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" [[package]] name = "uuid" -version = "1.17.0" +version = "1.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cf4199d1e5d15ddd86a694e4d0dffa9c323ce759fea589f00fef9d81cc1931d" +checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2" dependencies = [ "getrandom 0.3.3", "js-sys", diff --git a/nix/shells/development.nix b/nix/shells/development.nix index 3fc2c333..253882aa 100644 --- a/nix/shells/development.nix +++ b/nix/shells/development.nix @@ -5,7 +5,7 @@ { fenix, system, instantiatedPkgs, seekSdk }: let p = instantiatedPkgs // { - native = p.${system}; + native = instantiatedPkgs.${system}; }; seekSdkPath = seekSdk + "/Seek_Thermal_SDK_4.1.0.0"; # Gets the same rust toolchain that rustup would have used. @@ -64,7 +64,6 @@ in cargo-zigbuild # Used to cross compile rust dpkg # Used to test outputs of cargo-deb git-cliff # Conventional commit based release notes - sshpass # Non-interactive ssh password auth mdbook # Generates site for docs mdbook-mermaid # Adds mermaid support nixpkgs-fmt # Nix autoformatter @@ -73,6 +72,7 @@ in (python3.withPackages (ps: with ps; [ requests ])) + qemu squashfsTools # mksquashfs sshpass # Needed for orb-software/scripts taplo # toml autoformatter @@ -87,7 +87,10 @@ in # env variables ourselves and don't want nix overwriting them, so we # use the unwrapped version. pkg-config-unwrapped - ]) ++ [ + ]) ++ p.native.lib.lists.optionals p.native.stdenv.isLinux [ + p.native.guestfs-tools + p.native.passt + ] ++ [ rustToolchain rustPlatform.bindgenHook # Configures bindgen to use nix clang ] ++ p.native.lib.lists.optionals p.native.stdenv.isDarwin [ diff --git a/test-utils/Cargo.toml b/test-utils/Cargo.toml index 758c722f..66fe7a65 100644 --- a/test-utils/Cargo.toml +++ b/test-utils/Cargo.toml @@ -9,7 +9,12 @@ rust-version.workspace = true publish = false [dependencies] +blake3 = "1.8.2" +cmd_lib.workspace = true +fs4 = "0.13.1" nix = { workspace = true, features = ["socket"] } +regex = "1.11.2" tempfile.workspace = true testcontainers.workspace = true tokio = { workspace = true, features = ["full"] } +uuid = { version = "1.18.1", features = ["v4"] } diff --git a/test-utils/src/lib.rs b/test-utils/src/lib.rs index 9a08e034..a98d0c0e 100644 --- a/test-utils/src/lib.rs +++ b/test-utils/src/lib.rs @@ -1,2 +1,3 @@ pub mod async_bag; pub mod docker; +pub mod qemu; diff --git a/test-utils/src/qemu/base.rs b/test-utils/src/qemu/base.rs new file mode 100644 index 00000000..caa64253 --- /dev/null +++ b/test-utils/src/qemu/base.rs @@ -0,0 +1,51 @@ +use super::img::QemuImg; + +const NET_ENP0S3_SVC: &str = " +[Unit] +Description=Bring up enp0s3 static +Before=network.target + +[Service] +Type=oneshot +ExecStart=/usr/sbin/ip link set enp0s3 up +ExecStart=/usr/sbin/ip addr add 10.0.2.15/24 dev enp0s3 +ExecStart=/usr/sbin/ip route add default via 10.0.2.2 +RemainAfterExit=yes + +[Install] +WantedBy=multi-user.target +"; + +const RESOLVED_CONF: &str = " +[Resolve] +DNS=10.0.2.3 1.1.1.1 +FallbackDNS="; + +const SUDOERS: &str = "worldcoin ALL=(ALL) NOPASSWD:ALL"; + +const SSHD_CFG: &str = " +PasswordAuthentication yes +PermitEmptyPasswords yes +UsePAM yes +PubkeyAuthentication no +UseDNS no +GSSAPIAuthentication no +MaxStartups 100:30:200 +"; + +pub fn bullseye() -> QemuImg { + QemuImg::from_base("debian-11-generic-amd64.qcow2") + .write("/etc/systemd/system/net-enp0s3.service", NET_ENP0S3_SVC) + .run("systemctl enable net-enp0s3.service") + .run("systemctl disable systemd-resolved") + .write("/etc/systemd/resolved.conf.d/99-qemu.conf", RESOLVED_CONF) + .run("ln -sf /run/systemd/resolve/stub-resolv.conf /etc/resolv.conf") + .run("systemctl enable systemd-resolved") + .write("/etc/sudoers.d/worldcoin", SUDOERS) + .write("/etc/ssh/sshd_config.d/99-local.conf", SSHD_CFG) + .run("useradd -m -s /bin/bash -G sudo worldcoin") + .run("passwd -d worldcoin") + .pkgs(&["openssh-server", "iproute2", "network-manager"]) + .run("systemctl enable ssh") + .run("ssh-keygen -A") +} diff --git a/test-utils/src/qemu/fx.rs b/test-utils/src/qemu/fx.rs new file mode 100644 index 00000000..b7a68139 --- /dev/null +++ b/test-utils/src/qemu/fx.rs @@ -0,0 +1,95 @@ +use super::{img::QemuImg, instance::QemuInstance}; +use std::{ + fs, + path::{Path, PathBuf}, +}; + +pub fn run(workdir: impl AsRef, img: &QemuImg) -> QemuInstance { + if !s3::is_authed() { + panic!("\nplease authenticate with s3 before continuing\n"); + } + + let img_path = get_or_build_img(&workdir, img); + QemuInstance::start(workdir, img_path) +} + +fn get_or_build_img(workdir: impl AsRef, qemu_img: &QemuImg) -> PathBuf { + let workdir = workdir.as_ref(); + let workdir_str = workdir.to_str().unwrap(); + + let img = format!("{}.qcow2", qemu_img.to_hash()); + + let img_workdir_path = workdir.join(&img); + println!("checking for QemuImg in workdir: {img_workdir_path:?}"); + + if fs::exists(&img_workdir_path).unwrap() { + return img_workdir_path; + } + + println!( + "QemuImg does not exist. Looking for it in {}", + s3::VM_S3_PATH + ); + + if s3::get_vm(workdir_str, &img) { + return img_workdir_path; + } + + println!("QemuImg does not exist in S3, will build locally."); + + let base_img_path = workdir.join(qemu_img.base()); + println!("Looking for base image in workdir: {base_img_path:?}"); + + if !fs::exists(&base_img_path).unwrap() { + println!("Base image not found locally. Pulling from s3."); + if !s3::get_vm(workdir_str, qemu_img.base()) { + panic!( + "Could not find base image {} on S3. Nothing else to do.", + qemu_img.base() + ); + } + } + + println!("Building QemuImg."); + qemu_img.build(workdir) +} + +mod s3 { + use std::{fs::OpenOptions, path::Path}; + + use cmd_lib::run_cmd; + use fs4::fs_std::FileExt; + + pub const VM_S3_PATH: &str = "s3://worldcoin-orb-resources/virtual-machines"; + + pub fn is_authed() -> bool { + run_cmd!(aws sts get-caller-identity > /dev/null).is_ok() + } + + pub fn get_vm(workdir: impl AsRef, filename: &str) -> bool { + let workdir = workdir.as_ref(); + let workdir_str = workdir.to_str().unwrap(); + + let lock_path = workdir.join(format!("{filename}.lock")); + let file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(lock_path) + .unwrap(); + + file.lock_exclusive().unwrap(); + + if workdir.join(filename).exists() { + return true; + } + + run_cmd!(aws s3 cp $VM_S3_PATH/$filename $workdir_str).is_ok() + } + + #[allow(dead_code)] + pub fn upload_vm(workdir: &str, filename: &str) { + run_cmd!(aws s3 cp $workdir/$filename $VM_S3_PATH).unwrap(); + } +} diff --git a/test-utils/src/qemu/img.rs b/test-utils/src/qemu/img.rs new file mode 100644 index 00000000..6089786b --- /dev/null +++ b/test-utils/src/qemu/img.rs @@ -0,0 +1,206 @@ +use cmd_lib::run_cmd; +use fs4::fs_std::FileExt; +use std::{ + fmt, + fs::OpenOptions, + path::{Path, PathBuf}, + process::Command, +}; + +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct QemuImg { + base: String, + steps: Vec, +} + +#[derive(Debug, Clone, Eq, PartialEq)] +enum QemuStep { + Write { + guest_path: String, + contents: String, + }, + Package(String), + Run(String), +} + +impl fmt::Display for QemuStep { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + use QemuStep::*; + + match self { + Write { + guest_path, + contents, + } => write!(f, "{guest_path}:{contents}"), + + Package(pkg) => write!(f, "{pkg}"), + Run(cmd) => write!(f, "{cmd}"), + } + } +} + +impl QemuImg { + pub fn base(&self) -> &str { + &self.base + } + + /// A base qcow2 image to build upon. + pub fn from_base(guest_base: impl Into) -> Self { + Self { + base: guest_base.into(), + steps: vec![], + } + } + + pub fn to_hash(&self) -> String { + let mut h = blake3::Hasher::new(); + let mut update = |tag: &str, contents: &str| { + h.update(tag.as_bytes()); + h.update(&(contents.len() as u64).to_le_bytes()); + h.update(contents.as_bytes()); + }; + + update("t", "qemuimgv1"); + update("b", &self.base); + + use QemuStep::*; + for step in &self.steps { + match step { + Write { + guest_path, + contents, + } => { + update("wp", guest_path); + update("wc", contents); + } + + Package(pkg) => update("p", pkg), + Run(cmd) => update("r", cmd), + } + } + + h.finalize().to_hex().to_string() + } + + /// Ensures directory exists on guest, and writes to the filepath when image is being built. + pub fn write( + mut self, + guest_path: impl Into, + contents: impl Into, + ) -> Self { + self.steps.push(QemuStep::Write { + guest_path: guest_path.into(), + contents: contents.into(), + }); + self + } + + /// Installs a package on the guest using its package manager when image is being built. + pub fn pkg(mut self, pkg: impl Into) -> Self { + self.steps.push(QemuStep::Package(pkg.into())); + self + } + + /// Installs a package on the guest using apt when image is being built. + pub fn pkgs(mut self, pkgs: &[&str]) -> Self { + for pkg in pkgs { + self.steps.push(QemuStep::Package(pkg.to_string())); + } + + self + } + + /// Runs a command on the guest when image is being built. + pub fn run(mut self, guest_cmd: impl Into) -> Self { + self.steps.push(QemuStep::Run(guest_cmd.into())); + self + } + + pub fn build(&self, working_dir: impl AsRef) -> PathBuf { + let working_dir = working_dir.as_ref(); + let base_path = working_dir.join(&self.base).to_string_lossy().to_string(); + let hash = self.to_hash(); + let img = format!("{hash}.qcow2"); + let img_path = working_dir.join(img); + + let lock = format!("{hash}.lock"); + let lock_path = working_dir.join(lock); + let file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(lock_path) + .unwrap(); + + file.lock_exclusive().unwrap(); + + // if file exists by the time lock releases, it was most likely built by another thread or + // process while lock was being held + if img_path.exists() { + return img_path; + } + + run_cmd!(cp $base_path $img_path).unwrap(); + + let mut cmd = Command::new("virt-customize"); + cmd.args([ + "-a", + img_path.to_str().unwrap(), + // dont bloat image with apt and logs + "--run-command", + "set -eux; \ + mkdir -p /var/lib/apt/lists /var/cache/apt/archives /var/log; \ + mount -t tmpfs tmpfs /var/lib/apt/lists; \ + mount -t tmpfs tmpfs /var/cache/apt/archives; \ + mount -t tmpfs tmpfs /var/log; \ + apt-get update", + ]); + + for step in &self.steps { + use QemuStep::*; + match step { + Write { + guest_path, + contents, + } => { + cmd.args([ + "--run-command", + &format!("mkdir -p \"$(dirname {guest_path})\""), + ]); + + cmd.args([ + "--run-command", + &format!("cat > {guest_path} <<'EOF'\n{contents}\nEOF"), + ]); + } + + Package(pkg) => { + cmd.args(["--run-command", &format!("DEBIAN_FRONTEND=noninteractive apt-get -y install --no-install-recommends {pkg}")]); + } + + Run(c) => { + cmd.args(["--run-command", c]); + } + } + } + + cmd.args([ + "--run-command", + " + umount /var/lib/apt/lists; \ + umount /var/cache/apt/archives; \ + umount /var/log", + ]); + + let output = cmd.output().expect("failed to spawn virt-customize"); + if !output.status.success() { + let status = output.status.code().unwrap(); + let stderr = String::from_utf8_lossy(&output.stderr); + + panic!("virt-customize failed with status: {status}, stderr: {stderr}"); + } + + img_path + } +} diff --git a/test-utils/src/qemu/instance.rs b/test-utils/src/qemu/instance.rs new file mode 100644 index 00000000..92db9ab2 --- /dev/null +++ b/test-utils/src/qemu/instance.rs @@ -0,0 +1,133 @@ +use cmd_lib::{run_cmd, run_fun}; +use regex::Regex; +use std::{ + io::{BufRead, BufReader, Write}, + os::unix::net::UnixStream, + path::Path, + thread, + time::Duration, +}; +use uuid::Uuid; + +#[derive(Debug)] +pub struct QemuInstance { + id: String, + ssh_port: u16, +} + +impl Drop for QemuInstance { + fn drop(&mut self) { + self.kill(); + } +} + +impl QemuInstance { + pub fn start(working_dir: impl AsRef, img_path: impl AsRef) -> Self { + let working_dir = working_dir.as_ref(); + let img_path = img_path.as_ref().to_str().unwrap(); + let id = Uuid::new_v4().to_string(); + + let tmp_overlay_path = working_dir + .join(format!("{id}.qcow2")) + .to_string_lossy() + .to_string(); + + let qmp = working_dir + .join(format!("{id}.qmp")) + .to_string_lossy() + .to_string(); + + run_cmd! { + qemu-img create -f qcow2 -F qcow2 -b $img_path $tmp_overlay_path; + + qemu-system-x86_64 + -machine q35 -cpu max -m 512 + -accel tcg,thread=multi -smp 2 + -daemonize -display none + -name guest=$id,process=qemu-$id + -qmp unix:$qmp,server,nowait + -drive file=$tmp_overlay_path,if=virtio,format=qcow2 + -object rng-random,filename=/dev/urandom,id=rng0 + -device virtio-rng-pci,rng=rng0 + -nic user,model=virtio-net-pci,hostfwd=tcp:127.0.0.1:0-:22,ipv6=off + } + .unwrap(); + + println!("getting ssh port"); + let ssh_port = qmp_ssh_port(&qmp); + + println!("checking if guest is listening on ssh port {ssh_port}"); + for _ in 0..10 { + let result = run_cmd! { + ssh -p $ssh_port -o BatchMode=yes -o ConnectTimeout=5 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o GlobalKnownHostsFile=/dev/null + worldcoin@127.0.0.1 echo hello world + }; + + if result.is_ok() { + println!("guest ready"); + return Self { id, ssh_port }; + } + + thread::sleep(std::time::Duration::from_millis(10_000)); + } + + panic!("timed out when trying to reach vm through ssh on port {ssh_port}") + } + + fn kill(&self) { + let id = &self.id; + run_cmd!(pkill -f process=qemu-$id).unwrap(); + } + + pub fn copy(&self, host_path: impl AsRef, guest_path: impl AsRef) { + let host_path = host_path.as_ref().to_str().unwrap(); + let guest_path = guest_path.as_ref().to_str().unwrap(); + let port = self.ssh_port; + + run_cmd!{ + scp -O -P $port -o BatchMode=yes -o ConnectTimeout=5 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o GlobalKnownHostsFile=/dev/null + $host_path worldcoin@127.0.0.1:$guest_path + }.unwrap(); + } + + pub fn run(&self, cmd: &str) -> String { + let port = self.ssh_port; + run_fun! { + ssh -p $port -o BatchMode=yes -o ConnectTimeout=5 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o GlobalKnownHostsFile=/dev/null + worldcoin@127.0.0.1 $cmd + }.unwrap() + } +} + +pub fn qmp_ssh_port(qmp_path: &str) -> u16 { + let stream = UnixStream::connect(qmp_path).unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(3))) + .unwrap(); + + let mut writer = stream.try_clone().unwrap(); + let mut reader = BufReader::new(stream); + + let payload = concat!( + r#"{"execute":"qmp_capabilities"}"#, + "\n", + r#"{"execute":"human-monitor-command","arguments":{"command-line":"info usernet"} }"#, + "\n" + ); + + writer.write_all(payload.as_bytes()).unwrap(); + writer.flush().unwrap(); + + let re = + Regex::new(r#"127\.0\.0\.1[ :]+(\d+)(?:\s*->\s*|\s+)[0-9.]+\s+22"#).unwrap(); + + let mut line = String::new(); + while reader.read_line(&mut line).unwrap() > 0 { + if let Some(c) = re.captures(&line) { + return c[1].parse().unwrap(); + } + line.clear(); + } + + panic!("hostfwd :22 not found") +} diff --git a/test-utils/src/qemu/mod.rs b/test-utils/src/qemu/mod.rs new file mode 100644 index 00000000..1bb567d4 --- /dev/null +++ b/test-utils/src/qemu/mod.rs @@ -0,0 +1,4 @@ +pub mod base; +pub mod fx; +pub mod img; +pub mod instance; diff --git a/test-utils/tests/qemu.rs b/test-utils/tests/qemu.rs new file mode 100644 index 00000000..ac1e967a --- /dev/null +++ b/test-utils/tests/qemu.rs @@ -0,0 +1,28 @@ +use cmd_lib::run_cmd; +use test_utils::qemu::{self, base}; + +#[cfg_attr(target_os = "macos", test_with::no_env(GITHUB_ACTIONS))] +#[test] +fn run_in_parallel_one() { + let q = qemu::fx::run("/tmp", &base::bullseye()); + + run_cmd!(echo blabla > /tmp/hello).unwrap(); + q.copy("/tmp/hello", "/home/worldcoin/hello"); + + let guest_hello = q.run("cat /home/worldcoin/hello"); + + assert_eq!(guest_hello, "blabla"); +} + +#[cfg_attr(target_os = "macos", test_with::no_env(GITHUB_ACTIONS))] +#[test] +fn run_in_parallel_two() { + let q = qemu::fx::run("/tmp", &base::bullseye()); + + run_cmd!(echo blabla > /tmp/hello).unwrap(); + q.copy("/tmp/hello", "/home/worldcoin/hello"); + + let guest_hello = q.run("cat /home/worldcoin/hello"); + + assert_eq!(guest_hello, "blabla"); +}