apiserver: add --ebs-volumes and --prefer options to ephemeral-storage init command

Author: Jesse Anttila

packages/os/ephemeral-ebs-storage.rules

@@ -0,0 +1,12 @@
+# ephemeral storage links: /dev/disk/ephemeral-ebs
+
+ACTION=="remove", GOTO="ephemeral_ebs_storage_end"
+KERNEL!="nvme*", GOTO="ephemeral_ebs_storage_end"
+SUBSYSTEM!="block", GOTO="ephemeral_ebs_storage_end"
+ENV{DEVTYPE}!="disk", GOTO="ephemeral_ebs_storage_end"
+ATTRS{model}!="Amazon Elastic Block Store", GOTO="ephemeral_ebs_storage_end"
+
+# Create links for ephemeral EBS volumes that start with "xvdd".
+ENV{BOTTLEROCKET_DEVICE_TYPE}=="ephemeral", ENV{XVD_DEVICE_NAME}=="xvdd?", SYMLINK+="disk/ephemeral-ebs/$env{XVD_DEVICE_NAME}"
+
+LABEL="ephemeral_ebs_storage_end"

packages/os/os.spec

@@ -64,7 +64,8 @@ Source205: bootstrap-commands-tmpfiles.conf
 # 3xx sources: udev rules
 Source300: ephemeral-storage.rules
 Source301: ebs-volumes.rules
-Source302: supplemental-storage.rules
+Source302: ephemeral-ebs-storage.rules
+Source303: supplemental-storage.rules
 
 # 4xx sources: Bottlerocket licenses
 Source400: COPYRIGHT
@@ -727,7 +728,8 @@ install -p -m 0644 %{S:205} %{buildroot}%{_cross_tmpfilesdir}/bootstrap-commands
 install -d %{buildroot}%{_cross_udevrulesdir}
 install -p -m 0644 %{S:300} %{buildroot}%{_cross_udevrulesdir}/80-ephemeral-storage.rules
 install -p -m 0644 %{S:301} %{buildroot}%{_cross_udevrulesdir}/81-ebs-volumes.rules
-install -p -m 0644 %{S:302} %{buildroot}%{_cross_udevrulesdir}/82-supplemental-storage.rules
+install -p -m 0644 %{S:302} %{buildroot}%{_cross_udevrulesdir}/82-ephemeral-ebs-storage.rules
+install -p -m 0644 %{S:303} %{buildroot}%{_cross_udevrulesdir}/83-supplemental-storage.rules
 
 %cross_scan_attribution --clarify %{_builddir}/sources/clarify.toml \
     cargo --offline --locked %{_builddir}/sources/Cargo.toml
@@ -809,7 +811,8 @@ install -p -m 0644 %{S:400} %{S:401} %{S:402} %{buildroot}%{_cross_licensedir}
 %{_cross_bindir}/ghostdog
 %{_cross_udevrulesdir}/80-ephemeral-storage.rules
 %{_cross_udevrulesdir}/81-ebs-volumes.rules
-%{_cross_udevrulesdir}/82-supplemental-storage.rules
+%{_cross_udevrulesdir}/82-ephemeral-ebs-storage.rules
+%{_cross_udevrulesdir}/83-supplemental-storage.rules
 
 %files -n %{_cross_os}signpost
 %{_cross_bindir}/signpost

sources/api/apiserver/src/server/ephemeral_storage.rs

@@ -1,6 +1,6 @@
 //! The 'ephemeral_storage' module supports configuring and using local instance storage.
 
-use model::ephemeral_storage::Filesystem;
+use model::ephemeral_storage::{Filesystem, Preference};
 
 use snafu::{ensure, ResultExt};
 use std::collections::HashSet;
@@ -22,6 +22,8 @@ static EPHEMERAL_MNT: &str = ".ephemeral";
 /// Name of the device and its path from the MD driver
 static RAID_DEVICE_DIR: &str = "/dev/md/";
 static RAID_DEVICE_NAME: &str = "ephemeral";
+/// Symlink to ephemeral storage array or disk
+static EPHEMERAL_STORAGE_LINK: &str = "/dev/disk/ephemeral-storage";
 
 pub struct BindDirs {
     pub allowed_exact: HashSet<&'static str>,
@@ -32,14 +34,25 @@ pub struct BindDirs {
 /// initialize prepares the ephemeral storage for formatting and formats it.  For multiple disks
 /// preparation is the creation of a RAID0 array, for a single disk this is a no-op. The array or disk
 /// is then formatted with the specified filesystem (default=xfs) if not formatted already.
-pub fn initialize(fs: Option<Filesystem>, disks: Option<Vec<String>>) -> Result<()> {
+pub fn initialize(
+    fs: Option<Filesystem>,
+    disks: Option<Vec<String>>,
+    ebs_volumes: Option<Vec<String>>,
+    prefer: Option<Vec<Preference>>,
+) -> Result<()> {
     let known_disks = ephemeral_devices()?;
     let known_disks_hash = HashSet::<_>::from_iter(known_disks.iter());
+    let known_ebs_volumes = ephemeral_ebs_volumes()?;
+    let known_ebs_volumes_hash = HashSet::<_>::from_iter(known_ebs_volumes.iter());
 
-    let disks = match disks {
-        Some(disks) => {
-            // we have disks provided, so match them against the list of valid disks
-            for disk in &disks {
+    let any_specified = disks.as_ref().is_some_and(|x| !x.is_empty())
+        || ebs_volumes.as_ref().is_some_and(|x| !x.is_empty());
+
+    let disks = if any_specified {
+        // use all specified ephemeral disks and ebs volumes, if they're all valid
+        let mut selected_disks = vec![];
+        if let Some(d) = disks {
+            for disk in &d {
                 ensure!(
                     known_disks_hash.contains(disk),
                     error::InvalidParameterSnafu {
@@ -48,26 +61,57 @@ pub fn initialize(fs: Option<Filesystem>, disks: Option<Vec<String>>) -> Result<
                     }
                 )
             }
-            disks
+            selected_disks.extend(d);
         }
-        None => {
-            // if there are no disks specified, and none are available we treat the init as a
-            // no-op to allow "ephemeral-storage init"/"ephemeral-storage bind" to work on instances
-            // with and without ephemeral storage
-            if known_disks.is_empty() {
-                info!("no ephemeral disks found, skipping ephemeral storage initialization");
-                return Ok(());
+
+        if let Some(e) = ebs_volumes {
+            for ebs_volume in &e {
+                ensure!(
+                    known_ebs_volumes_hash.contains(ebs_volume),
+                    error::InvalidParameterSnafu {
+                        parameter: "ebs_volumes",
+                        reason: format!("unknown ebs volume {ebs_volume:?}"),
+                    }
+                )
+            }
+            selected_disks.extend(e);
+        }
+        selected_disks
+    } else {
+        // if there are no specified disks, use preference list to find a non-empty set of disks
+        let preferences = prefer.unwrap_or_else(|| {
+            vec![Preference {
+                ephemeral_disk: true,
+                ebs_volume: false,
+            }]
+        });
+
+        let mut disks = vec![];
+        for preference in preferences {
+            if preference.ephemeral_disk {
+                disks.extend(&known_disks);
+            }
+            if preference.ebs_volume {
+                disks.extend(&known_ebs_volumes);
+            }
+            if !disks.is_empty() {
+                break;
             }
-            // no disks specified, so use the default
-            known_disks
         }
+        if disks.is_empty() {
+            // no disks were specified and none of the preferences produced any disks
+            // this is special-cased as a no-op
+            info!("no ephemeral disks found, skipping ephemeral storage initialization");
+            return Ok(());
+        }
+        disks.into_iter().cloned().collect()
     };
 
     ensure!(
         !disks.is_empty(),
         error::InvalidParameterSnafu {
             parameter: "disks",
-            reason: "no local ephemeral disks specified",
+            reason: "no valid local ephemeral disks or ebs volumes specified",
         }
     );
 
@@ -101,22 +145,21 @@ pub fn initialize(fs: Option<Filesystem>, disks: Option<Vec<String>>) -> Result<
         );
     }
 
+    // Create link to formatted device for use in `bind`
+    std::os::unix::fs::symlink(&device_name, EPHEMERAL_STORAGE_LINK)
+        .context(error::DiskSymlinkFailureSnafu {})?;
+
     Ok(())
 }
 
 /// binds the specified directories to the pre-configured array, creating those directories if
 /// they do not exist.
 pub fn bind(variant: &str, dirs: Vec<String>) -> Result<()> {
-    let device_name = match ephemeral_devices()?.len() {
-        // handle the no local instance storage case
-        0 => {
-            info!("no ephemeral disks found, skipping ephemeral storage binding");
-            return Ok(());
-        }
-        // If there is only one device, use that
-        1 => ephemeral_devices()?.first().expect("non-empty").clone(),
-        _ => format!("{RAID_DEVICE_DIR}{RAID_DEVICE_NAME}"),
-    };
+    let device_name = EPHEMERAL_STORAGE_LINK;
+    if !std::fs::exists(device_name).is_ok_and(|x| x) {
+        info!("ephemeral storage not initialized, skipping binding");
+        return Ok(());
+    }
 
     // Normalize input by trimming trailing "/"
     let dirs: Vec<String> = dirs
@@ -150,7 +193,7 @@ pub fn bind(variant: &str, dirs: Vec<String>) -> Result<()> {
     info!("mounting {:?} as {:?}", device_name, mount_point);
     let output = Command::new(MOUNT)
         .args([
-            OsString::from(device_name.clone()),
+            OsString::from(device_name),
             OsString::from(mount_point.as_os_str()),
         ])
         .output()
@@ -289,6 +332,29 @@ pub fn ephemeral_devices() -> Result<Vec<String>> {
     Ok(filenames)
 }
 
+/// ephemeral_ebs_volumes returns the full path name to the ebs volumes in /dev/disk/ephemeral-ebs
+pub fn ephemeral_ebs_volumes() -> Result<Vec<String>> {
+    const EPHEMERAL_EBS_PATH: &str = "/dev/disk/ephemeral-ebs";
+    let mut filenames = Vec::new();
+    // for instances without ebs volumes attached, we don't error and just return an empty vector so
+    // it can be handled gracefully
+    if fs::metadata(EPHEMERAL_EBS_PATH).is_err() {
+        return Ok(filenames);
+    }
+
+    let entries =
+        std::fs::read_dir(EPHEMERAL_EBS_PATH).context(error::DiscoverEbsVolumesSnafu {
+            path: String::from(EPHEMERAL_EBS_PATH),
+        })?;
+    for entry in entries {
+        let entry = entry.context(error::DiscoverEbsVolumesSnafu {
+            path: String::from(EPHEMERAL_EBS_PATH),
+        })?;
+        filenames.push(entry.path().into_os_string().to_string_lossy().to_string());
+    }
+    Ok(filenames)
+}
+
 /// allowed_bind_dirs returns a set of the directories that can be bound to ephemeral storage, which
 /// varies based on the variant, a set of the prefixes of directories that are allowed to be bound.
 /// and a set of substrings that are disallowed in the directory name.
@@ -385,6 +451,12 @@ pub mod error {
             path: String,
         },
 
+        #[snafu(display("Failed to discover ebs volumes from {}: {}", path, source))]
+        DiscoverEbsVolumes {
+            source: std::io::Error,
+            path: String,
+        },
+
         #[snafu(display("Failed to mount {} to {}: {}", what, dest, String::from_utf8_lossy(output.stderr.as_slice())))]
         MountArrayFailure {
             what: String,

sources/api/apiserver/src/server/mod.rs

@@ -726,8 +726,13 @@ async fn get_fips_report(query: web::Query<HashMap<String, String>>) -> Result<H
 
 /// Configure ephemeral storage (raid & format, or just format for single disk)
 async fn initialize_ephemeral_storage(cfg: web::Json<Init>) -> Result<HttpResponse> {
-    ephemeral_storage::initialize(cfg.0.filesystem, cfg.0.disks)
-        .context(error::EphemeralInitializeSnafu {})?;
+    ephemeral_storage::initialize(
+        cfg.0.filesystem,
+        cfg.0.disks,
+        cfg.0.ebs_volumes,
+        cfg.0.prefer,
+    )
+    .context(error::EphemeralInitializeSnafu {})?;
     Ok(HttpResponse::NoContent().finish()) // 204
 }
 /// Bind directories to ephemeral storage (mount array, bind, and unmount)

sources/models/src/ephemeral_storage.rs

@@ -18,11 +18,58 @@ impl Display for Filesystem {
     }
 }
 
+/// Storage type preferences for ephemeral storage
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct Preference {
+    pub ephemeral_disk: bool,
+    pub ebs_volume: bool,
+}
+impl Display for Preference {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        let mut first = false;
+        let mut write = |s| {
+            if first {
+                first = false;
+            } else {
+                f.write_str("+")?;
+            }
+            f.write_str(s)
+        };
+        if self.ephemeral_disk {
+            write("ephemeral-disk")?;
+        }
+        if self.ebs_volume {
+            write("ebs-volume")?;
+        }
+        Ok(())
+    }
+}
+impl<'a> TryFrom<&'a str> for Preference {
+    type Error = &'a str;
+
+    fn try_from(value: &'a str) -> Result<Self, Self::Error> {
+        let mut preference = Self {
+            ephemeral_disk: false,
+            ebs_volume: false,
+        };
+        for i in value.split("+") {
+            match i {
+                "ephemeral-disk" => preference.ephemeral_disk = true,
+                "ebs-volume" => preference.ebs_volume = true,
+                x => return Err(x),
+            }
+        }
+        Ok(preference)
+    }
+}
+
 /// Initialize ephemeral storage
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct Init {
     pub filesystem: Option<Filesystem>,
     pub disks: Option<Vec<String>>,
+    pub ebs_volumes: Option<Vec<String>>,
+    pub prefer: Option<Vec<Preference>>,
 }
 
 /// Bind directories to configured ephemeral storage