Merge pull request #2 from spinupdev/feat/jail-userfaultfd

feat: recreate /dev/userfaultfd inside the jail

Author: aaqaishtyaq

ARCHITECTURE.md

@@ -19,7 +19,7 @@ JSON manifest -> manifest validation -> Linux launch workflow
   or credential details.
 - `src/linux/jail.rs` stages the root, applies declared bind mounts, resolves
   allow-listed VFIO character identities, performs `pivot_root`, and creates
-  only the KVM/TUN/entropy and declared VFIO nodes required by CH.
+  only the KVM/TUN/entropy/userfaultfd and declared VFIO nodes required by CH.
 - `src/linux/cgroup.rs` owns cgroup-v2 discovery, controller delegation through
   `cgroup.subtree_control`, limit writes, and process attachment.
 - `src/linux/process.rs` owns namespaces, resource limits, descriptor and
@@ -54,7 +54,9 @@ Intentional differences:
   process, logs, and durable PID directly;
 - the API socket is created by CH inside the jail, not passed as a listener FD
   or bind mounted from the host;
-- Firecracker-specific userfaultfd support is not exposed; and
+- Cloud Hypervisor OnDemand restore needs `/dev/userfaultfd` inside the jail
+  (recreated from the host character identity, owned by the unprivileged VMM).
+  Firecracker's external uffd-handler is out of scope for this launcher; and
 - VFIO groups are supplied by the host allocator; this launcher validates the
   boundary but does not discover devices or decide assignment policy.
 

Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "cloud-hypervisor-jailer"
-version = "0.1.10"
+version = "0.1.11"
 edition = "2024"
 rust-version = "1.85"
 description = "Cloud Hypervisor sandbox launcher for Depot"

Cargo.toml

@@ -22,7 +22,9 @@ On Linux, `launch` requires root and then:
 - mounts only declared non-symlink sources;
 - joins a pre-created network namespace when requested;
 - configures the declared cgroup-v2 values and resource limits;
-- creates jailed KVM, TUN, and entropy device nodes;
+- creates jailed KVM, TUN, entropy, and (when the host has it) userfaultfd
+  device nodes so Cloud Hypervisor OnDemand restore can create a uffd without
+  `vm.unprivileged_userfaultfd=1`;
 - recreates only explicitly declared canonical VFIO control/group character
   devices, without bind-mounting host `/dev` or changing host device ownership;
 - creates a PID namespace when requested;
@@ -35,7 +37,7 @@ On Linux, `launch` requires root and then:
 flowchart LR
   O["Host orchestrator"] -->|"versioned JSON manifest"| V["validate"]
   V -->|"pure checks"| L["launch as root"]
-  L --> J["jail\nmount namespace • bind mounts • pivot_root • KVM/TUN/VFIO"]
+  L --> J["jail\nmount namespace • bind mounts • pivot_root • KVM/TUN/userfaultfd/VFIO"]
   L --> C["cgroup v2\ncontroller delegation • limits • lease"]
   L --> P["process\nnetns/PID ns • rlimits • FD/env cleanup • UID/GID"]
   J --> CH["Cloud Hypervisor\n--seccomp true"]

README.md

@@ -70,6 +70,31 @@ pub(super) fn resolve_devices(manifest: &Manifest) -> Result<Vec<ResolvedDevice>
         .collect::<Result<Vec<_>>>()
 }
 
+const USERFAULTFD_PATH: &str = "/dev/userfaultfd";
+
+/// Resolve the host `/dev/userfaultfd` identity before pivot, if the node
+/// has one. Cloud Hypervisor OnDemand restore tries this device first, then
+/// the `userfaultfd(2)` syscall. Recreating the character device inside the
+/// jail (owned by the unprivileged VMM) is the narrow grant upstream
+/// recommends, instead of `vm.unprivileged_userfaultfd=1` for every process.
+pub(super) fn resolve_userfaultfd() -> Result<Option<ResolvedDevice>> {
+    match fs::symlink_metadata(USERFAULTFD_PATH) {
+        Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(None),
+        Err(err) => Err(err).context("stat /dev/userfaultfd"),
+        Ok(metadata) => {
+            if metadata.file_type().is_symlink() || !metadata.file_type().is_char_device() {
+                bail!("/dev/userfaultfd is not a character device");
+            }
+            let device_id = metadata.rdev();
+            Ok(Some(ResolvedDevice {
+                destination: std::path::PathBuf::from("dev/userfaultfd"),
+                major: libc::major(device_id) as u32,
+                minor: libc::minor(device_id) as u32,
+            }))
+        }
+    }
+}
+
 fn resolve_device(device: &Device) -> Result<ResolvedDevice> {
     let metadata = fs::symlink_metadata(&device.source)
         .with_context(|| format!("stat device source {}", device.source.display()))?;
@@ -112,7 +137,12 @@ pub(super) fn pivot_into_jail(root: &Path) -> Result<()> {
     syscall_ok(unsafe { libc::rmdir(old_root.as_ptr()) }).context("remove old root")
 }
 
-pub(super) fn create_device_nodes(uid: u32, gid: u32, devices: &[ResolvedDevice]) -> Result<()> {
+pub(super) fn create_device_nodes(
+    uid: u32,
+    gid: u32,
+    devices: &[ResolvedDevice],
+    userfaultfd: Option<&ResolvedDevice>,
+) -> Result<()> {
     fs::create_dir_all("/dev/net").context("create jailed dev directory")?;
     if !devices.is_empty() {
         fs::create_dir_all("/dev/vfio").context("create jailed VFIO directory")?;
@@ -123,6 +153,12 @@ pub(super) fn create_device_nodes(uid: u32, gid: u32, devices: &[ResolvedDevice]
     // Expose only this non-blocking entropy device; guest workloads never
     // receive the host /dev filesystem.
     create_character_device(Path::new("/dev/urandom"), 1, 9)?;
+    if let Some(device) = userfaultfd {
+        create_character_device(Path::new("/dev/userfaultfd"), device.major, device.minor)
+            .context("create jailed /dev/userfaultfd")?;
+        chown_path(Path::new("/dev/userfaultfd"), uid, gid)
+            .context("chown jailed /dev/userfaultfd")?;
+    }
     for device in devices {
         let destination = Path::new("/").join(&device.destination);
         create_character_device(&destination, device.major, device.minor)
@@ -280,6 +316,7 @@ fn mount_call(source: Option<&Path>, destination: &Path, flags: libc::c_ulong) -
 #[cfg(test)]
 mod tests {
     use super::artifact_mode;
+    use super::resolve_userfaultfd;
 
     #[test]
     fn mounted_artifact_modes_are_minimally_permissive() {
@@ -288,4 +325,19 @@ mod tests {
         assert_eq!(artifact_mode(true, true), 0o500);
         assert_eq!(artifact_mode(true, false), 0o700);
     }
+
+    #[test]
+    fn resolve_userfaultfd_is_optional_when_the_host_has_no_device() {
+        match resolve_userfaultfd() {
+            Ok(None) => {}
+            Ok(Some(device)) => {
+                assert_eq!(
+                    device.destination,
+                    std::path::PathBuf::from("dev/userfaultfd")
+                );
+                assert!(device.major > 0 || device.minor > 0);
+            }
+            Err(err) => panic!("resolve_userfaultfd: {err}"),
+        }
+    }
 }

src/linux/jail.rs

@@ -21,6 +21,7 @@ pub(crate) fn launch(manifest: &Manifest) -> Result<()> {
         .transpose()
         .context("open network namespace")?;
     let devices = jail::resolve_devices(manifest)?;
+    let userfaultfd = jail::resolve_userfaultfd()?;
 
     jail::prepare_root(manifest)?;
     jail::enter_mount_namespace()?;
@@ -32,7 +33,7 @@ pub(crate) fn launch(manifest: &Manifest) -> Result<()> {
     // effective capabilities needed for pivot_root and device setup.
     process::drop_capability_bounding_set()?;
     jail::pivot_into_jail(&manifest.root)?;
-    jail::create_device_nodes(manifest.uid, manifest.gid, &devices)?;
+    jail::create_device_nodes(manifest.uid, manifest.gid, &devices, userfaultfd.as_ref())?;
     if let Some(netns) = netns {
         process::join_network_namespace(netns.as_raw_fd())?;
     }