diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5fd23c0ad..22c42e249 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -106,6 +106,7 @@ jobs: sudo find /ostree/repo/objects -name '*.file' -type f | while read f; do sudo fsverity measure $f >/dev/null done + # Test that we can build documentation docs: runs-on: ubuntu-24.04 @@ -197,12 +198,20 @@ jobs: - name: Unit and container integration tests run: just test-container - - name: Run TMT tests + - name: Run TMT integration tests run: | if [ "${{ matrix.variant }}" = "composefs-sealeduki-sdboot" ]; then just test-composefs else - just test-tmt + just test-tmt integration + fi + - name: Run TMT test about bootc install on coreos + run: | + # Only test fedora-43 on fedora-coreos:testing-devel + if [ "${{ matrix.test_os }}" = "fedora-43" ] && [ "${{ matrix.variant }}" = "ostree" ]; then + just test-tmt-on-coreos plan-bootc-install-on-coreos + else + echo "skipped" fi - name: Archive TMT logs diff --git a/Justfile b/Justfile index 748e9bb35..7d6b68c38 100644 --- a/Justfile +++ b/Justfile @@ -43,6 +43,9 @@ buildargs := base_buildargs + " --secret=id=secureboot_key,src=target/test-secur # Args for build-sealed (no base arg, it sets that itself) sealed_buildargs := "--build-arg=variant=" + variant + " --secret=id=secureboot_key,src=target/test-secureboot/db.key --secret=id=secureboot_cert,src=target/test-secureboot/db.crt" +# Needed by bootc install on ostree +fedora-coreos := "quay.io/fedora/fedora-coreos:testing-devel" + # The default target: build the container image from current sources. # Note commonly you might want to override the base image via e.g. # `just build --build-arg=base=quay.io/fedora/fedora-bootc:42` @@ -161,6 +164,13 @@ _build-upgrade-image: test-tmt-nobuild *ARGS: cargo xtask run-tmt --env=BOOTC_variant={{variant}} --upgrade-image={{integration_upgrade_img}} {{integration_img}} {{ARGS}} +# Test bootc install on coreos (FCOS) +# Assume the localhost/bootc-integration image is up to date, and just run tests. +# BOOTC_target is the bootc-integration, that will be used for bootc install +test-tmt-on-coreos *ARGS: + @if ! podman image exists {{integration_img}}; then just build-integration-test-image; fi + cargo xtask run-tmt --env=BOOTC_variant={{variant}} --env=BOOTC_target={{integration_img}}:latest {{fedora-coreos}} {{ARGS}} + # Cleanup all test VMs created by tmt tests tmt-vm-cleanup: bcvk libvirt rm --stop --force --label bootc.test=1 @@ -175,6 +185,7 @@ test-container: build-units build-integration-test-image clean-local-images: podman images --filter "label={{testimage_label}}" podman images --filter "label={{testimage_label}}" --format "{{{{.ID}}" | xargs -r podman rmi -f + podman rmi {{fedora-coreos}} -f # Print the container image reference for a given short $ID-VERSION_ID for NAME # and 'base' or 'buildroot-base' for TYPE (base image type) diff --git a/crates/lib/src/bootloader.rs b/crates/lib/src/bootloader.rs index 6126cef9e..c09763c7b 100644 --- a/crates/lib/src/bootloader.rs +++ b/crates/lib/src/bootloader.rs @@ -4,12 +4,13 @@ use anyhow::{anyhow, bail, Context, Result}; use bootc_utils::CommandRunExt; use camino::Utf8Path; use cap_std_ext::cap_std::fs::Dir; +use cap_std_ext::dirext::CapStdExtDirExt; use fn_error_context::context; use bootc_blockdev::{Partition, PartitionTable}; use bootc_mount as mount; -use crate::bootc_composefs::boot::mount_esp; +use crate::bootc_composefs::boot::{get_sysroot_parent_dev, mount_esp}; use crate::{discoverable_partition_specification, utils}; /// The name of the mountpoint for efi (as a subdirectory of /boot, or at the toplevel) @@ -26,6 +27,36 @@ pub(crate) fn esp_in(device: &PartitionTable) -> Result<&Partition> { .ok_or(anyhow::anyhow!("ESP not found in partition table")) } +/// Get esp partition node based on the root dir +pub(crate) fn get_esp_partition_node(root: &Dir) -> Result> { + let device = get_sysroot_parent_dev(&root)?; + let base_partitions = bootc_blockdev::partitions_of(Utf8Path::new(&device))?; + let esp = base_partitions.find_partition_of_esp()?; + if let Some(esp) = esp { + return Ok(Some(esp.node.clone())); + }; + Ok(None) +} + +// Mount esp part at /boot/efi +pub(crate) fn mount_esp_part(root: &Dir, root_path: &Utf8Path, physical_root: &Dir) -> Result<()> { + let efi_path = Utf8Path::new("boot").join(crate::bootloader::EFI_DIR); + if let Some(esp_fd) = root + .open_dir_optional(&efi_path) + .context("Opening /boot/efi")? + { + if let Some(false) = esp_fd.is_mountpoint(".")? { + tracing::debug!("Not a mountpoint: /boot/efi"); + // On ostree env, should use /target/sysroot because of composefs + if let Some(esp_part) = get_esp_partition_node(&physical_root)? { + bootc_mount::mount(&esp_part, &root_path.join(&efi_path))?; + tracing::debug!("Mounted {esp_part} at /boot/efi"); + } + } + } + Ok(()) +} + /// Determine if the invoking environment contains bootupd, and if there are bootupd-based /// updates in the target root. #[context("Querying for bootupd")] diff --git a/crates/lib/src/install.rs b/crates/lib/src/install.rs index ba35ef47f..b248015c3 100644 --- a/crates/lib/src/install.rs +++ b/crates/lib/src/install.rs @@ -769,6 +769,17 @@ async fn initialize_ostree_root(state: &State, root_setup: &RootSetup) -> Result // Another implementation: https://github.com/coreos/coreos-assembler/blob/3cd3307904593b3a131b81567b13a4d0b6fe7c90/src/create_disk.sh#L295 crate::lsm::ensure_dir_labeled(rootfs_dir, "", Some("/".into()), 0o755.into(), sepolicy)?; + // If we're installing alongside existing ostree and there's a separate boot partition, + // we need to mount it to the sysroot's /boot so ostree can write bootloader entries there + if has_ostree && root_setup.boot.is_some() { + if let Some(boot) = &root_setup.boot { + let source_boot = &boot.source; + let target_boot = root_setup.physical_root_path.join(BOOT); + tracing::debug!("Mount {source_boot} to {target_boot} on ostree"); + bootc_mount::mount(source_boot, &target_boot)?; + } + } + // And also label /boot AKA xbootldr, if it exists if rootfs_dir.try_exists("boot")? { crate::lsm::ensure_dir_labeled(rootfs_dir, "boot", None, 0o755.into(), sepolicy)?; @@ -1095,6 +1106,8 @@ pub(crate) struct RootSetup { pub(crate) physical_root_path: Utf8PathBuf, /// Directory file descriptor for the above physical root. pub(crate) physical_root: Dir, + /// Target root path /target. + pub(crate) target_root_path: Option, pub(crate) rootfs_uuid: Option, /// True if we should skip finalizing skip_finalize: bool, @@ -1549,7 +1562,10 @@ async fn install_with_sysroot( Bootloader::Grub => { crate::bootloader::install_via_bootupd( &rootfs.device_info, - &rootfs.physical_root_path, + &rootfs + .target_root_path + .clone() + .unwrap_or(rootfs.physical_root_path.clone()), &state.config_opts, Some(&deployment_path.as_str()), )?; @@ -1896,30 +1912,61 @@ fn remove_all_in_dir_no_xdev(d: &Dir, mount_err: bool) -> Result<()> { anyhow::Ok(()) } +#[context("Removing boot directory content except loader dir on ostree")] +fn remove_all_except_loader_dirs(bootdir: &Dir, is_ostree: bool) -> Result<()> { + let entries = bootdir + .entries() + .context("Reading boot directory entries")?; + + for entry in entries { + let entry = entry.context("Reading directory entry")?; + let file_name = entry.file_name(); + let file_name = if let Some(n) = file_name.to_str() { + n + } else { + anyhow::bail!("Invalid non-UTF8 filename: {file_name:?} in /boot"); + }; + + // TODO: Preserve basically everything (including the bootloader entries + // on non-ostree) by default until the very end of the install. And ideally + // make the "commit" phase an optional step after. + if is_ostree && file_name.starts_with("loader") { + continue; + } + + let etype = entry.file_type()?; + if etype == FileType::dir() { + // Open the directory and remove its contents + if let Some(subdir) = bootdir.open_dir_noxdev(&file_name)? { + remove_all_in_dir_no_xdev(&subdir, false) + .with_context(|| format!("Removing directory contents: {}", file_name))?; + bootdir.remove_dir(&file_name)?; + } + } else { + bootdir + .remove_file_optional(&file_name) + .with_context(|| format!("Removing file: {}", file_name))?; + } + } + Ok(()) +} + #[context("Removing boot directory content")] fn clean_boot_directories(rootfs: &Dir, is_ostree: bool) -> Result<()> { let bootdir = crate::utils::open_dir_remount_rw(rootfs, BOOT.into()).context("Opening /boot")?; - if is_ostree { - // On ostree systems, the boot directory already has our desired format, we should only - // remove the bootupd-state.json file to avoid bootupctl complaining it already exists. - bootdir - .remove_file_optional("bootupd-state.json") - .context("removing bootupd-state.json")?; - } else { - // This should not remove /boot/efi note. - remove_all_in_dir_no_xdev(&bootdir, false).context("Emptying /boot")?; - // TODO: Discover the ESP the same way bootupd does it; we should also - // support not wiping the ESP. - if ARCH_USES_EFI { - if let Some(efidir) = bootdir - .open_dir_optional(crate::bootloader::EFI_DIR) - .context("Opening /boot/efi")? - { - remove_all_in_dir_no_xdev(&efidir, false) - .context("Emptying EFI system partition")?; - } + // This should not remove /boot/efi note. + remove_all_except_loader_dirs(&bootdir, is_ostree).context("Emptying /boot")?; + + // TODO: Discover the ESP the same way bootupd does it; we should also + // support not wiping the ESP. + if ARCH_USES_EFI { + if let Some(efidir) = bootdir + .open_dir_optional(crate::bootloader::EFI_DIR) + .context("Opening /boot/efi")? + { + remove_all_in_dir_no_xdev(&efidir, false).context("Emptying EFI system partition")?; } } @@ -2056,6 +2103,18 @@ pub(crate) async fn install_to_filesystem( .context("Mounting host / to {ALONGSIDE_ROOT_MOUNT}")?; } + let target_root_path = fsopts.root_path.clone(); + // Get a file descriptor for the root path /target + let target_rootfs_fd = + Dir::open_ambient_dir(&target_root_path, cap_std::ambient_authority()) + .with_context(|| format!("Opening target root directory {target_root_path}"))?; + + tracing::debug!("Target root filesystem: {target_root_path}"); + + if let Some(false) = target_rootfs_fd.is_mountpoint(".")? { + anyhow::bail!("Not a mountpoint: {target_root_path}"); + } + // Check that the target is a directory { let root_path = &fsopts.root_path; @@ -2069,10 +2128,7 @@ pub(crate) async fn install_to_filesystem( // Check to see if this happens to be the real host root if !fsopts.acknowledge_destructive { - let root_path = &fsopts.root_path; - let rootfs_fd = Dir::open_ambient_dir(root_path, cap_std::ambient_authority()) - .with_context(|| format!("Opening target root directory {root_path}"))?; - warn_on_host_root(&rootfs_fd)?; + warn_on_host_root(&target_rootfs_fd)?; } // If we're installing to an ostree root, then find the physical root from @@ -2088,7 +2144,8 @@ pub(crate) async fn install_to_filesystem( }; // Get a file descriptor for the root path - let rootfs_fd = { + // It will be /target/sysroot on ostree OS, or will be /target + let rootfs_fd = if is_already_ostree { let root_path = &fsopts.root_path; let rootfs_fd = Dir::open_ambient_dir(&fsopts.root_path, cap_std::ambient_authority()) .with_context(|| format!("Opening target root directory {root_path}"))?; @@ -2099,6 +2156,8 @@ pub(crate) async fn install_to_filesystem( anyhow::bail!("Not a mountpoint: {root_path}"); } rootfs_fd + } else { + target_rootfs_fd.try_clone()? }; match fsopts.replace { @@ -2108,7 +2167,18 @@ pub(crate) async fn install_to_filesystem( tokio::task::spawn_blocking(move || remove_all_in_dir_no_xdev(&rootfs_fd, true)) .await??; } - Some(ReplaceMode::Alongside) => clean_boot_directories(&rootfs_fd, is_already_ostree)?, + Some(ReplaceMode::Alongside) => { + // On existing ostree OS like FCOS, esp is not mounted after booted, + // need to find esp and mount before clean + if ARCH_USES_EFI { + crate::bootloader::mount_esp_part( + &target_rootfs_fd, + &target_root_path, + &rootfs_fd, + )?; + } + clean_boot_directories(&target_rootfs_fd, is_already_ostree)? + } None => require_empty_rootdir(&rootfs_fd)?, } @@ -2153,7 +2223,7 @@ pub(crate) async fn install_to_filesystem( let boot_is_mount = { let root_dev = rootfs_fd.dir_metadata()?.dev(); - let boot_dev = rootfs_fd + let boot_dev = target_rootfs_fd .symlink_metadata_optional(BOOT)? .ok_or_else(|| { anyhow!("No /{BOOT} directory found in root; this is is currently required") @@ -2164,9 +2234,10 @@ pub(crate) async fn install_to_filesystem( }; // Find the UUID of /boot because we need it for GRUB. let boot_uuid = if boot_is_mount { - let boot_path = fsopts.root_path.join(BOOT); + let boot_path = target_root_path.join(BOOT); + tracing::debug!("boot_path={boot_path}"); let u = bootc_mount::inspect_filesystem(&boot_path) - .context("Inspecting /{BOOT}")? + .with_context(|| format!("Inspecting /{BOOT}"))? .uuid .ok_or_else(|| anyhow!("No UUID found for /{BOOT}"))?; Some(u) @@ -2247,6 +2318,7 @@ pub(crate) async fn install_to_filesystem( device_info, physical_root_path: fsopts.root_path, physical_root: rootfs_fd, + target_root_path: Some(target_root_path.clone()), rootfs_uuid: inspect.uuid.clone(), boot, kargs, diff --git a/crates/lib/src/install/baseline.rs b/crates/lib/src/install/baseline.rs index 40a537e11..faca9483d 100644 --- a/crates/lib/src/install/baseline.rs +++ b/crates/lib/src/install/baseline.rs @@ -491,6 +491,7 @@ pub(crate) fn install_create_rootfs( device_info, physical_root_path, physical_root, + target_root_path: None, rootfs_uuid: Some(root_uuid.to_string()), boot, kargs, diff --git a/crates/xtask/src/tmt.rs b/crates/xtask/src/tmt.rs index 3a97fe4c1..bf97d0a82 100644 --- a/crates/xtask/src/tmt.rs +++ b/crates/xtask/src/tmt.rs @@ -137,6 +137,25 @@ fn build_firmware_args(sh: &Shell, image: &str) -> Result> { Ok(r) } +/// Detect VARIANT_ID from container image by reading os-release +/// Returns string like "coreos" or empty +#[context("Detecting distro from image")] +fn detect_variantid_from_image(sh: &Shell, image: &str) -> Result> { + let variant_id = cmd!( + sh, + "podman run --rm {image} bash -c '. /usr/lib/os-release && echo $VARIANT_ID'" + ) + .read() + .context("Failed to run image as container to detect distro")?; + + let variant_id = variant_id.trim(); + if variant_id.is_empty() { + return Ok(None); + } + + Ok(Some(variant_id.to_string())) +} + /// Check if a distro supports --bind-storage-ro /// CentOS 9 lacks systemd.extra-unit.* support required for bind-storage-ro fn distro_supports_bind_storage_ro(distro: &str) -> bool { @@ -269,6 +288,9 @@ pub(crate) fn run_tmt(sh: &Shell, args: &RunTmtArgs) -> Result<()> { // Detect distro from the image let distro = detect_distro_from_image(sh, image)?; + // Detect VARIANT_ID from the image + // As this can not be empty value in context, use "unknown" instead + let variant_id = detect_variantid_from_image(sh, image)?.unwrap_or("unknown".to_string()); let context = args .context @@ -276,11 +298,15 @@ pub(crate) fn run_tmt(sh: &Shell, args: &RunTmtArgs) -> Result<()> { .map(|v| format!("--context={}", v)) .chain(std::iter::once(format!("--context=running_env=image_mode"))) .chain(std::iter::once(format!("--context=distro={}", distro))) + .chain(std::iter::once(format!( + "--context=VARIANT_ID={variant_id}" + ))) .collect::>(); let preserve_vm = args.preserve_vm; println!("Using bcvk image: {}", image); println!("Detected distro: {}", distro); + println!("Detected VARIANT_ID: {variant_id}"); let firmware_args = build_firmware_args(sh, image)?; @@ -387,7 +413,16 @@ pub(crate) fn run_tmt(sh: &Shell, args: &RunTmtArgs) -> Result<()> { distro ); } - + // Add --filesystem=xfs by default on fedora-coreos + // Add --bind-storage-ro if supported + if variant_id == "coreos" { + if distro.starts_with("fedora") { + opts.push("--filesystem=xfs".to_string()); + } + if supports_bind_storage_ro { + opts.push(BCVK_OPT_BIND_STORAGE_RO.to_string()); + } + } opts }; diff --git a/tmt/plans/tests-install.fmf b/tmt/plans/tests-install.fmf new file mode 100644 index 000000000..7d7cb795e --- /dev/null +++ b/tmt/plans/tests-install.fmf @@ -0,0 +1,28 @@ +discover: + how: fmf +execute: + how: tmt +# Because of the two issues, run tmt on github runner directly failed +# - selinux disabled on ubuntu (https://github.com/teemtee/tmt/issues/3364) +# - uefi firmware is not supported (https://github.com/teemtee/tmt/issues/4203) +provision: + how: virtual + hardware: + boot: + method: uefi + image: fedora-coreos-next + user: root + memory: 4096 + disk: 20 + +/plan-bootc-install-on-coreos: + summary: Execute bootc install on ostree OS + adjust: + - when: VARIANT_ID != coreos + enabled: false + because: this needs to start an ostree OS firstly + discover: + how: fmf + test: + - /tmt/tests/install/test-bootc-install-on-coreos + extra-try_bind_storage: true diff --git a/tmt/tests/examples/test-on-ostree/test-install-on-ostree.sh b/tmt/tests/examples/test-on-ostree/test-install-on-ostree.sh new file mode 100644 index 000000000..f5a13a6f5 --- /dev/null +++ b/tmt/tests/examples/test-on-ostree/test-install-on-ostree.sh @@ -0,0 +1,56 @@ +# number: 50 +# extra: +# try_bind_storage: true +# tmt: +# summary: Test bootc install on ostree OS +# duration: 30m +# adjust: +# - when: VARIANT_ID != coreos +# enabled: false +# because: this needs to start an ostree OS firstly +# +#!/bin/bash +set -eux + +echo "Testing bootc install on ostree" + +# BOOTC_target is integration image +[ -n "$BOOTC_target" ] + +if [ "$TMT_REBOOT_COUNT" -eq 0 ]; then + echo "Running before first reboot" + pwd + ls -l + # Verify testing on ostree OS + if [ ! -f "/run/ostree-booted" ]; then + echo "Should be ostree OS" + exit 1 + fi + podman image exists ${BOOTC_target} + # Run bootc install using the same stateroot for shared /var + stateroot=$(bootc status --json | jq -r .status.booted.ostree.stateroot) + + # Need bind mount for /run/host-container-storage + podman run --rm --privileged \ + -v /dev:/dev \ + -v /run/host-container-storage:/run/host-container-storage:ro \ + -v /:/target \ + --pid=host \ + --security-opt label=type:unconfined_t \ + ${BOOTC_target} \ + env BOOTC_BOOTLOADER_DEBUG=1 STORAGE_OPTS=additionalimagestore=/run/host-container-storage \ + bootc install to-existing-root \ + --stateroot=${stateroot} \ + --skip-fetch-check \ + --acknowledge-destructive \ + --karg=console=ttyS0,115200n8 + + bootc status + tmt-reboot +elif [ "$TMT_REBOOT_COUNT" -eq 1 ]; then + echo 'After the reboot' + booted=$(bootc status --json | jq -r .status.booted.image.image.image) + [ ${booted} == ${BOOTC_target} ] +fi + +echo "Run bootc install on ostree OS successfully" diff --git a/tmt/tests/install.fmf b/tmt/tests/install.fmf new file mode 100644 index 000000000..6e16ba2bd --- /dev/null +++ b/tmt/tests/install.fmf @@ -0,0 +1,8 @@ +/test-bootc-install-on-coreos: + summary: Test bootc install on coreos + duration: 30m + adjust: + - when: VARIANT_ID != coreos + enabled: false + because: this needs to start an ostree OS firstly + test: bash examples/test-on-ostree/test-install-on-ostree.sh