Merge pull request #1 from spinupdev/codex/vfio-device-allowlist

feat: allow-list jailed VFIO devices

Author: aaqaishtyaq

ARCHITECTURE.md

@@ -17,8 +17,9 @@ JSON manifest -> manifest validation -> Linux launch workflow
   its pure invariants.
 - `src/linux/mod.rs` owns operation order. It has no low-level mount, cgroup,
   or credential details.
-- `src/linux/jail.rs` stages the root, applies declared bind mounts, performs
-  `pivot_root`, and creates only the KVM/TUN device nodes required by CH.
+- `src/linux/jail.rs` stages the root, applies declared bind mounts, resolves
+  allow-listed VFIO character identities, performs `pivot_root`, and creates
+  only the KVM/TUN/entropy and declared VFIO nodes required by CH.
 - `src/linux/cgroup.rs` owns cgroup-v2 discovery, controller delegation through
   `cgroup.subtree_control`, limit writes, and process attachment.
 - `src/linux/process.rs` owns namespaces, resource limits, descriptor and
@@ -38,8 +39,10 @@ Adopted or strengthened here:
 - strict versioned manifest rather than caller-provided arbitrary arguments;
 - bounded machine IDs, non-root target identity, path validation, and forced
   CH seccomp;
-- mount and PID namespaces, `pivot_root`, netns join, KVM/TUN nodes, rlimits,
-  cgroup-v2 limits, and descriptor/environment sanitization;
+- mount and PID namespaces, `pivot_root`, netns join, KVM/TUN/entropy nodes,
+  rlimits, cgroup-v2 limits, and descriptor/environment sanitization;
+- exact per-machine VFIO control/IOMMU-group allow-lists whose character
+  identities are recreated inside the jail instead of exposing host `/dev`;
 - cgroup-v2 controller availability checks and recursive delegation before a
   leaf cgroup is configured; and
 - `close_range` with an `ENOSYS` fallback, rather than an unconditional
@@ -52,8 +55,8 @@ Intentional differences:
 - the API socket is created by CH inside the jail, not passed as a listener FD
   or bind mounted from the host;
 - Firecracker-specific userfaultfd support is not exposed; and
-- `/dev/urandom` is not created because CH uses host randomness through the
-  kernel, not a jailed device path.
+- VFIO groups are supplied by the host allocator; this launcher validates the
+  boundary but does not discover devices or decide assignment policy.
 
 ## Remaining hardening work
 

Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "cloud-hypervisor-jailer"
-version = "0.1.6"
+version = "0.1.10"
 edition = "2024"
 rust-version = "1.85"
 description = "Cloud Hypervisor sandbox launcher for Depot"

Cargo.toml

@@ -22,7 +22,9 @@ On Linux, `launch` requires root and then:
 - mounts only declared non-symlink sources;
 - joins a pre-created network namespace when requested;
 - configures the declared cgroup-v2 values and resource limits;
-- creates jailed KVM and TUN device nodes;
+- creates jailed KVM, TUN, and entropy device nodes;
+- recreates only explicitly declared canonical VFIO control/group character
+  devices, without bind-mounting host `/dev` or changing host device ownership;
 - creates a PID namespace when requested;
 - clears inherited environment and non-standard file descriptors;
 - drops the complete capability bounding set, sets `no_new_privs`, and changes
@@ -33,7 +35,7 @@ On Linux, `launch` requires root and then:
 flowchart LR
   O["Host orchestrator"] -->|"versioned JSON manifest"| V["validate"]
   V -->|"pure checks"| L["launch as root"]
-  L --> J["jail\nmount namespace • bind mounts • pivot_root • KVM/TUN"]
+  L --> J["jail\nmount namespace • bind mounts • pivot_root • KVM/TUN/VFIO"]
   L --> C["cgroup v2\ncontroller delegation • limits • lease"]
   L --> P["process\nnetns/PID ns • rlimits • FD/env cleanup • UID/GID"]
   J --> CH["Cloud Hypervisor\n--seccomp true"]
@@ -94,6 +96,14 @@ in [`src/manifest.rs`](src/manifest.rs) for the current canonical contract. Trea
 paths and arguments as host-orchestrator-controlled inputs; this is not a
 safe interface for tenant-provided configuration.
 
+VFIO passthrough is opt-in per manifest. `devices` may contain only the exact
+control path `/dev/vfio/vfio` and canonical numeric IOMMU-group paths such as
+`/dev/vfio/42`; each destination must be the identical sandbox-relative path.
+Group entries require the control device. The jailer verifies every source is
+a real character device before pivoting and recreates its major/minor identity
+inside the private jail. Arbitrary host devices, symlinks, path aliases,
+destination remapping, duplicate nodes, and mounts over `dev/` are rejected.
+
 ## Status
 
 The launcher is tested for manifest validation and compiled/tested on native

README.md

@@ -26,6 +26,10 @@ release used by the consuming orchestrator:
    for reconciler retry.
 5. Create/restore a CH snapshot through the host orchestrator; verify the
    jailer does not broaden storage or device visibility.
+6. On a dedicated VFIO host, pass one complete IOMMU group and verify the VMM
+   sees only `/dev/vfio/vfio` plus that numeric group. Also verify arbitrary
+   devices, a second unassigned group, symlinks, and mounts over `/dev` fail
+   before Cloud Hypervisor execs.
 
 The workflow is deliberately manual until a hardened dedicated runner exists.
 Never run it on a shared developer host or a runner that also contains tenant

docs/testing.md

@@ -3,12 +3,14 @@
 use std::env;
 use std::ffi::CString;
 use std::fs::{self, OpenOptions};
+use std::os::unix::fs::FileTypeExt;
+use std::os::unix::fs::MetadataExt;
 use std::os::unix::fs::{OpenOptionsExt, PermissionsExt};
 use std::path::Path;
 
 use anyhow::{Context, Result, bail};
 
-use crate::manifest::{Manifest, Mount};
+use crate::manifest::{Device, Manifest, Mount};
 
 use super::util::{c_path, syscall_ok};
 
@@ -49,6 +51,50 @@ pub(super) fn mount_resources(manifest: &Manifest) -> Result<()> {
     Ok(())
 }
 
+#[derive(Debug)]
+pub(super) struct ResolvedDevice {
+    destination: std::path::PathBuf,
+    major: u32,
+    minor: u32,
+}
+
+/// Resolve the device identity while the host `/dev` tree is still visible.
+/// Manifest validation has already constrained every path to canonical VFIO
+/// names; this second check proves each selected entry is a real character
+/// device rather than trusting a regular file at an allowed-looking path.
+pub(super) fn resolve_devices(manifest: &Manifest) -> Result<Vec<ResolvedDevice>> {
+    manifest
+        .devices
+        .iter()
+        .map(resolve_device)
+        .collect::<Result<Vec<_>>>()
+}
+
+fn resolve_device(device: &Device) -> Result<ResolvedDevice> {
+    let metadata = fs::symlink_metadata(&device.source)
+        .with_context(|| format!("stat device source {}", device.source.display()))?;
+    if metadata.file_type().is_symlink() || !metadata.file_type().is_char_device() {
+        bail!(
+            "device source is not a character device: {}",
+            device.source.display()
+        );
+    }
+    let canonical = fs::canonicalize(&device.source)
+        .with_context(|| format!("canonicalize device source {}", device.source.display()))?;
+    if canonical != device.source {
+        bail!(
+            "device source changed during resolution: {}",
+            device.source.display()
+        );
+    }
+    let device_id = metadata.rdev();
+    Ok(ResolvedDevice {
+        destination: device.destination.0.clone(),
+        major: libc::major(device_id) as u32,
+        minor: libc::minor(device_id) as u32,
+    })
+}
+
 pub(super) fn pivot_into_jail(root: &Path) -> Result<()> {
     mount_call(Some(root), root, libc::MS_BIND | libc::MS_REC)?;
     env::set_current_dir(root).context("enter jail root")?;
@@ -66,14 +112,24 @@ pub(super) fn pivot_into_jail(root: &Path) -> Result<()> {
     syscall_ok(unsafe { libc::rmdir(old_root.as_ptr()) }).context("remove old root")
 }
 
-pub(super) fn create_device_nodes(uid: u32, gid: u32) -> Result<()> {
+pub(super) fn create_device_nodes(uid: u32, gid: u32, devices: &[ResolvedDevice]) -> Result<()> {
     fs::create_dir_all("/dev/net").context("create jailed dev directory")?;
+    if !devices.is_empty() {
+        fs::create_dir_all("/dev/vfio").context("create jailed VFIO directory")?;
+    }
     create_character_device(Path::new("/dev/kvm"), 10, 232)?;
     create_character_device(Path::new("/dev/net/tun"), 10, 200)?;
     // Cloud Hypervisor's default virtio-rng device reads from /dev/urandom.
     // Expose only this non-blocking entropy device; guest workloads never
     // receive the host /dev filesystem.
     create_character_device(Path::new("/dev/urandom"), 1, 9)?;
+    for device in devices {
+        let destination = Path::new("/").join(&device.destination);
+        create_character_device(&destination, device.major, device.minor)
+            .with_context(|| format!("create jailed device {}", destination.display()))?;
+        chown_path(&destination, uid, gid)
+            .with_context(|| format!("chown jailed device {}", destination.display()))?;
+    }
     for path in [
         Path::new("/"),
         Path::new("/dev/kvm"),

src/linux/jail.rs

@@ -20,6 +20,7 @@ pub(crate) fn launch(manifest: &Manifest) -> Result<()> {
         .map(File::open)
         .transpose()
         .context("open network namespace")?;
+    let devices = jail::resolve_devices(manifest)?;
 
     jail::prepare_root(manifest)?;
     jail::enter_mount_namespace()?;
@@ -31,7 +32,7 @@ pub(crate) fn launch(manifest: &Manifest) -> Result<()> {
     // effective capabilities needed for pivot_root and device setup.
     process::drop_capability_bounding_set()?;
     jail::pivot_into_jail(&manifest.root)?;
-    jail::create_device_nodes(manifest.uid, manifest.gid)?;
+    jail::create_device_nodes(manifest.uid, manifest.gid, &devices)?;
     if let Some(netns) = netns {
         process::join_network_namespace(netns.as_raw_fd())?;
     }