diff --git a/crates/lib/src/cli.rs b/crates/lib/src/cli.rs index 6ab4bdab2..ec69a185b 100644 --- a/crates/lib/src/cli.rs +++ b/crates/lib/src/cli.rs @@ -149,6 +149,14 @@ pub(crate) struct SwitchOpts { #[clap(long)] pub(crate) retain: bool, + /// Use unified storage path to pull images (experimental) + /// + /// When enabled, this uses bootc's container storage (/usr/lib/bootc/storage) to pull + /// the image first, then imports it from there. This is the same approach used for + /// logically bound images. + #[clap(long = "experimental-unified-storage", hide = true)] + pub(crate) unified_storage_exp: bool, + /// Target image to use for the next boot. pub(crate) target: String, @@ -445,6 +453,11 @@ pub(crate) enum ImageOpts { /// this will make the image accessible via e.g. `podman run localhost/bootc` and for builds. target: Option, }, + /// Re-pull the currently booted image into the bootc-owned container storage. + /// + /// This onboards the system to the unified storage path so that future + /// upgrade/switch operations can read from the bootc storage directly. + SetUnified, /// Copy a container image from the default `containers-storage:` to the bootc-owned container storage. PullFromDefaultStorage { /// The image to pull @@ -948,7 +961,15 @@ async fn upgrade( } } } else { - let fetched = crate::deploy::pull(repo, imgref, None, opts.quiet, prog.clone()).await?; + // Auto-detect whether to use unified storage based on image presence in bootc storage + let use_unified = crate::deploy::image_exists_in_unified_storage(storage, imgref).await; + + let fetched = if use_unified { + crate::deploy::pull_unified(repo, imgref, None, opts.quiet, prog.clone(), storage, None) + .await? + } else { + crate::deploy::pull(repo, imgref, None, opts.quiet, prog.clone()).await? + }; let staged_digest = staged_image.map(|s| s.digest().expect("valid digest in status")); let fetched_digest = &fetched.manifest_digest; tracing::debug!("staged: {staged_digest:?}"); @@ -1062,7 +1083,21 @@ async fn switch_ostree( let new_spec = RequiredHostSpec::from_spec(&new_spec)?; - let fetched = crate::deploy::pull(repo, &target, None, opts.quiet, prog.clone()).await?; + // Determine whether to use unified storage path. + // If explicitly requested via flag, use unified storage directly. + // Otherwise, auto-detect based on whether the image exists in bootc storage. + let use_unified = if opts.unified_storage_exp { + true + } else { + crate::deploy::image_exists_in_unified_storage(storage, &target).await + }; + + let fetched = if use_unified { + crate::deploy::pull_unified(repo, &target, None, opts.quiet, prog.clone(), storage, None) + .await? + } else { + crate::deploy::pull(repo, &target, None, opts.quiet, prog.clone()).await? + }; if !opts.retain { // By default, we prune the previous ostree ref so it will go away after later upgrades @@ -1422,7 +1457,9 @@ async fn run_from_opt(opt: Opt) -> Result<()> { tracing::debug!("Computing digest of {iid}"); if !host_container_store.try_exists()? { - anyhow::bail!("Must be readonly mount of host container store: {host_container_store}"); + anyhow::bail!( + "Must be readonly mount of host container store: {host_container_store}" + ); } // And ensure we're finding the image in the host storage let mut cmd = Command::new("skopeo"); @@ -1460,6 +1497,7 @@ async fn run_from_opt(opt: Opt) -> Result<()> { ImageOpts::CopyToStorage { source, target } => { crate::image::push_entrypoint(source.as_deref(), target.as_deref()).await } + ImageOpts::SetUnified => crate::image::set_unified_entrypoint().await, ImageOpts::PullFromDefaultStorage { image } => { let storage = get_storage().await?; storage @@ -1539,7 +1577,10 @@ async fn run_from_opt(opt: Opt) -> Result<()> { let mut w = SplitStreamWriter::new(&cfs, None, Some(testdata_digest)); w.write_inline(testdata); let object = cfs.write_stream(w, Some("testobject"))?.to_hex(); - assert_eq!(object, "5d94ceb0b2bb3a78237e0a74bc030a262239ab5f47754a5eb2e42941056b64cb21035d64a8f7c2f156e34b820802fa51884de2b1f7dc3a41b9878fc543cd9b07"); + assert_eq!( + object, + "5d94ceb0b2bb3a78237e0a74bc030a262239ab5f47754a5eb2e42941056b64cb21035d64a8f7c2f156e34b820802fa51884de2b1f7dc3a41b9878fc543cd9b07" + ); Ok(()) } // We don't depend on fsverity-utils today, so re-expose some helpful CLI tools. diff --git a/crates/lib/src/deploy.rs b/crates/lib/src/deploy.rs index 031700fc1..ae647a289 100644 --- a/crates/lib/src/deploy.rs +++ b/crates/lib/src/deploy.rs @@ -4,8 +4,8 @@ use std::collections::HashSet; use std::io::{BufRead, Write}; +use std::process::Command; -use anyhow::Ok; use anyhow::{anyhow, Context, Result}; use bootc_kernel_cmdline::utf8::CmdlineOwned; use cap_std::fs::{Dir, MetadataExt}; @@ -93,6 +93,17 @@ pub(crate) async fn new_importer( Ok(imp) } +/// Wrapper for pulling a container image with a custom proxy config (e.g. for unified storage). +pub(crate) async fn new_importer_with_config( + repo: &ostree::Repo, + imgref: &ostree_container::OstreeImageReference, + config: ostree_ext::containers_image_proxy::ImageProxyConfig, +) -> Result { + let mut imp = ostree_container::store::ImageImporter::new(repo, imgref, config).await?; + imp.require_bootable(); + Ok(imp) +} + pub(crate) fn check_bootc_label(config: &ostree_ext::oci_spec::image::ImageConfiguration) { if let Some(label) = labels_of_config(config).and_then(|labels| labels.get(crate::metadata::BOOTC_COMPAT_LABEL)) @@ -316,6 +327,18 @@ pub(crate) async fn prune_container_store(sysroot: &Storage) -> Result<()> { for deployment in deployments { let bound = crate::boundimage::query_bound_images_for_deployment(ostree, &deployment)?; all_bound_images.extend(bound.into_iter()); + // Also include the host image itself + // Note: Use just the image name (not the full transport:image format) because + // podman's image names don't include the transport prefix. + if let Some(host_image) = crate::status::boot_entry_from_deployment(ostree, &deployment)? + .image + .map(|i| i.image) + { + all_bound_images.push(crate::boundimage::BoundImage { + image: host_image.image.clone(), + auth_file: None, + }); + } } // Convert to a hashset of just the image names let image_names = HashSet::from_iter(all_bound_images.iter().map(|img| img.image.as_str())); @@ -381,6 +404,180 @@ pub(crate) async fn prepare_for_pull( Ok(PreparedPullResult::Ready(Box::new(prepared_image))) } +/// Check whether the image exists in bootc's unified container storage. +/// +/// This is used for auto-detection: if the image already exists in bootc storage +/// (e.g., from a previous `bootc image set-unified` or LBI pull), we can use +/// the unified storage path for faster imports. +/// +/// Returns true if the image exists in bootc storage. +pub(crate) async fn image_exists_in_unified_storage( + store: &Storage, + imgref: &ImageReference, +) -> bool { + let imgstore = match store.get_ensure_imgstore() { + Ok(s) => s, + Err(e) => { + tracing::warn!("Failed to access bootc storage: {e}; falling back to standard pull"); + return false; + } + }; + + let image_ref_str = imgref.to_transport_image(); + match imgstore.exists(&image_ref_str).await { + Ok(v) => v, + Err(e) => { + tracing::warn!( + "Failed to check bootc storage for image: {e}; falling back to standard pull" + ); + false + } + } +} + +/// Unified approach: Use bootc's CStorage to pull the image, then prepare from containers-storage. +/// This reuses the same infrastructure as LBIs. +/// +/// The `sysroot_path` parameter specifies the path to the sysroot where bootc storage is located. +/// During install, this should be the path to the target disk's mount point. +/// During upgrade/switch on a running system, pass `None` to use the default `/sysroot`. +pub(crate) async fn prepare_for_pull_unified( + repo: &ostree::Repo, + imgref: &ImageReference, + target_imgref: Option<&OstreeImageReference>, + store: &Storage, + sysroot_path: Option<&camino::Utf8Path>, +) -> Result { + // Get or initialize the bootc container storage (same as used for LBIs) + let imgstore = store.get_ensure_imgstore()?; + + let image_ref_str = imgref.to_transport_image(); + + // Always pull to ensure we have the latest image, whether from a remote + // registry or a locally rebuilt image + tracing::info!( + "Unified pull: pulling from transport '{}' to bootc storage", + &imgref.transport + ); + + // Pull the image to bootc storage using the same method as LBIs + // Show a spinner since podman pull can take a while and doesn't output progress + let pull_msg = format!("Pulling {} to bootc storage", &image_ref_str); + async_task_with_spinner(&pull_msg, async move { + imgstore + .pull(&image_ref_str, crate::podstorage::PullMode::Always) + .await + }) + .await?; + + // Now create a containers-storage reference to read from bootc storage + tracing::info!("Unified pull: now importing from containers-storage transport"); + let containers_storage_imgref = ImageReference { + transport: "containers-storage".to_string(), + image: imgref.image.clone(), + signature: imgref.signature.clone(), + }; + let ostree_imgref = OstreeImageReference::from(containers_storage_imgref); + + // Configure the importer to use bootc storage as an additional image store + let mut config = ostree_ext::containers_image_proxy::ImageProxyConfig::default(); + let mut cmd = Command::new("skopeo"); + // Use the actual physical path to bootc storage + // During install, this is the target disk's mount point; otherwise default to /sysroot + let sysroot_base = sysroot_path + .map(|p| p.to_string()) + .unwrap_or_else(|| "/sysroot".to_string()); + let storage_path = format!( + "{}/{}", + sysroot_base, + crate::podstorage::CStorage::subpath() + ); + crate::podstorage::set_additional_image_store(&mut cmd, &storage_path); + config.skopeo_cmd = Some(cmd); + + // Use the preparation flow with the custom config + let mut imp = new_importer_with_config(repo, &ostree_imgref, config).await?; + if let Some(target) = target_imgref { + imp.set_target(target); + } + let prep = match imp.prepare().await? { + PrepareResult::AlreadyPresent(c) => { + println!("No changes in {imgref:#} => {}", c.manifest_digest); + return Ok(PreparedPullResult::AlreadyPresent(Box::new((*c).into()))); + } + PrepareResult::Ready(p) => p, + }; + check_bootc_label(&prep.config); + if let Some(warning) = prep.deprecated_warning() { + ostree_ext::cli::print_deprecated_warning(warning).await; + } + ostree_ext::cli::print_layer_status(&prep); + let layers_to_fetch = prep.layers_to_fetch().collect::>>()?; + + // Log that we're importing a new image from containers-storage + const PULLING_NEW_IMAGE_ID: &str = "6d5e4f3a2b1c0d9e8f7a6b5c4d3e2f1a0"; + tracing::info!( + message_id = PULLING_NEW_IMAGE_ID, + bootc.image.reference = &imgref.image, + bootc.image.transport = "containers-storage", + bootc.original_transport = &imgref.transport, + bootc.status = "importing_from_storage", + "Importing image from bootc storage: {}", + ostree_imgref + ); + + let prepared_image = PreparedImportMeta { + imp, + n_layers_to_fetch: layers_to_fetch.len(), + layers_total: prep.all_layers().count(), + bytes_to_fetch: layers_to_fetch.iter().map(|(l, _)| l.layer.size()).sum(), + bytes_total: prep.all_layers().map(|l| l.layer.size()).sum(), + digest: prep.manifest_digest.clone(), + prep, + }; + + Ok(PreparedPullResult::Ready(Box::new(prepared_image))) +} + +/// Unified pull: Use podman to pull to containers-storage, then read from there +/// +/// The `sysroot_path` parameter specifies the path to the sysroot where bootc storage is located. +/// For normal upgrade/switch operations, pass `None` to use the default `/sysroot`. +pub(crate) async fn pull_unified( + repo: &ostree::Repo, + imgref: &ImageReference, + target_imgref: Option<&OstreeImageReference>, + quiet: bool, + prog: ProgressWriter, + store: &Storage, + sysroot_path: Option<&camino::Utf8Path>, +) -> Result> { + match prepare_for_pull_unified(repo, imgref, target_imgref, store, sysroot_path).await? { + PreparedPullResult::AlreadyPresent(existing) => { + // Log that the image was already present (Debug level since it's not actionable) + const IMAGE_ALREADY_PRESENT_ID: &str = "5c4d3e2f1a0b9c8d7e6f5a4b3c2d1e0f9"; + tracing::debug!( + message_id = IMAGE_ALREADY_PRESENT_ID, + bootc.image.reference = &imgref.image, + bootc.image.transport = &imgref.transport, + bootc.status = "already_present", + "Image already present: {}", + imgref + ); + Ok(existing) + } + PreparedPullResult::Ready(prepared_image_meta) => { + // To avoid duplicate success logs, pass a containers-storage imgref to the importer + let cs_imgref = ImageReference { + transport: "containers-storage".to_string(), + image: imgref.image.clone(), + signature: imgref.signature.clone(), + }; + pull_from_prepared(&cs_imgref, quiet, prog, *prepared_image_meta).await + } + } +} + #[context("Pulling")] pub(crate) async fn pull_from_prepared( imgref: &ImageReference, @@ -430,18 +627,21 @@ pub(crate) async fn pull_from_prepared( let imgref_canonicalized = imgref.clone().canonicalize()?; tracing::debug!("Canonicalized image reference: {imgref_canonicalized:#}"); - // Log successful import completion - const IMPORT_COMPLETE_JOURNAL_ID: &str = "4d3e2f1a0b9c8d7e6f5a4b3c2d1e0f9a8"; - - tracing::info!( - message_id = IMPORT_COMPLETE_JOURNAL_ID, - bootc.image.reference = &imgref.image, - bootc.image.transport = &imgref.transport, - bootc.manifest_digest = import.manifest_digest.as_ref(), - bootc.ostree_commit = &import.merge_commit, - "Successfully imported image: {}", - imgref - ); + // Log successful import completion (skip if using unified storage to avoid double logging) + let is_unified_path = imgref.transport == "containers-storage"; + if !is_unified_path { + const IMPORT_COMPLETE_JOURNAL_ID: &str = "4d3e2f1a0b9c8d7e6f5a4b3c2d1e0f9a8"; + + tracing::info!( + message_id = IMPORT_COMPLETE_JOURNAL_ID, + bootc.image.reference = &imgref.image, + bootc.image.transport = &imgref.transport, + bootc.manifest_digest = import.manifest_digest.as_ref(), + bootc.ostree_commit = &import.merge_commit, + "Successfully imported image: {}", + imgref + ); + } if let Some(msg) = ostree_container::store::image_filtered_content_warning(&import.filtered_files) @@ -1051,7 +1251,7 @@ pub(crate) fn fixup_etc_fstab(root: &Dir) -> Result<()> { } // Read the input, and atomically write a modified version - root.atomic_replace_with(fstab_path, move |mut w| { + root.atomic_replace_with(fstab_path, move |mut w| -> Result<()> { for line in fd.lines() { let line = line?; if !edit_fstab_line(&line, &mut w)? { diff --git a/crates/lib/src/image.rs b/crates/lib/src/image.rs index 584f4c46d..a4c4c3b1b 100644 --- a/crates/lib/src/image.rs +++ b/crates/lib/src/image.rs @@ -14,12 +14,26 @@ use serde::Serialize; use crate::{ boundimage::query_bound_images, cli::{ImageListFormat, ImageListType}, - podstorage::{ensure_floating_c_storage_initialized, CStorage}, + podstorage::CStorage, + utils::async_task_with_spinner, }; /// The name of the image we push to containers-storage if nothing is specified. const IMAGE_DEFAULT: &str = "localhost/bootc"; +/// Check if an image exists in the default containers-storage (podman storage). +/// +/// TODO: Using exit codes to check image existence is not ideal. We should use +/// the podman HTTP API via bollard () or similar +/// to properly communicate with podman and get structured responses. This would +/// also enable proper progress monitoring during pull operations. +async fn image_exists_in_host_storage(image: &str) -> Result { + use tokio::process::Command as AsyncCommand; + let mut cmd = AsyncCommand::new("podman"); + cmd.args(["image", "exists", image]); + Ok(cmd.status().await?.success()) +} + #[derive(Clone, Serialize, ValueEnum)] enum ImageListTypeColumn { Host, @@ -128,6 +142,9 @@ pub(crate) async fn list_entrypoint( /// Implementation of `bootc image push-to-storage`. #[context("Pushing image")] pub(crate) async fn push_entrypoint(source: Option<&str>, target: Option<&str>) -> Result<()> { + // Initialize floating c_storage early - needed for container operations + crate::podstorage::ensure_floating_c_storage_initialized(); + let transport = Transport::ContainerStorage; let sysroot = crate::cli::get_storage().await?; let ostree = sysroot.get_ostree()?; @@ -140,7 +157,6 @@ pub(crate) async fn push_entrypoint(source: Option<&str>, target: Option<&str>) name: target.to_owned(), } } else { - ensure_floating_c_storage_initialized(); ImageReference { transport: Transport::ContainerStorage, name: IMAGE_DEFAULT.to_string(), @@ -181,3 +197,177 @@ pub(crate) async fn imgcmd_entrypoint( cmd.args(args); cmd.run_capture_stderr() } + +/// Re-pull the currently booted image into the bootc-owned container storage. +/// +/// This onboards the system to unified storage for host images so that +/// upgrade/switch can use the unified path automatically when the image is present. +#[context("Setting unified storage for booted image")] +pub(crate) async fn set_unified_entrypoint() -> Result<()> { + // Initialize floating c_storage early - needed for container operations + crate::podstorage::ensure_floating_c_storage_initialized(); + + let sysroot = crate::cli::get_storage().await?; + set_unified(&sysroot).await +} + +/// Inner implementation of set_unified that accepts a storage reference. +#[context("Setting unified storage for booted image")] +pub(crate) async fn set_unified(sysroot: &crate::store::Storage) -> Result<()> { + let ostree = sysroot.get_ostree()?; + let repo = &ostree.repo(); + + // Discover the currently booted image reference. + // get_status_require_booted validates that we have a booted deployment with an image. + let (_booted_ostree, _deployments, host) = crate::status::get_status_require_booted(ostree)?; + + // Use the booted deployment's image from the status we just retrieved. + // get_status_require_booted guarantees host.status.booted is Some. + let booted_entry = host + .status + .booted + .as_ref() + .ok_or_else(|| anyhow::anyhow!("No booted deployment found"))?; + let image_status = booted_entry + .image + .as_ref() + .ok_or_else(|| anyhow::anyhow!("Booted deployment is not from a container image"))?; + + // Extract the ImageReference from the ImageStatus + let imgref = &image_status.image; + + // Canonicalize for pull display only, but we want to preserve original pullspec + let imgref_display = imgref.clone().canonicalize()?; + + // Pull the image from its original source into bootc storage using LBI machinery + let imgstore = sysroot.get_ensure_imgstore()?; + + const SET_UNIFIED_JOURNAL_ID: &str = "1a0b9c8d7e6f5a4b3c2d1e0f9a8b7c6d"; + tracing::info!( + message_id = SET_UNIFIED_JOURNAL_ID, + bootc.image.reference = &imgref_display.image, + bootc.image.transport = &imgref_display.transport, + "Re-pulling booted image into bootc storage via unified path: {}", + imgref_display + ); + + // Determine the appropriate source for pulling the image into bootc storage. + // + // Case 1: If source transport is containers-storage, the image was installed from + // local container storage. Copy it from the default containers-storage to + // the bootc storage if it exists there, if not pull from ostree store. + // Case 2: Otherwise, pull from the specified transport (usually a remote registry). + let is_containers_storage = imgref.transport() == Transport::ContainerStorage; + + if is_containers_storage { + tracing::info!( + "Source transport is containers-storage; checking if image exists in host storage" + ); + + // Check if the image already exists in the default containers-storage. + // This can happen if someone did a local build (e.g., podman build) and + // we don't want to overwrite it with an export from ostree. + let image_exists = image_exists_in_host_storage(&imgref.image).await?; + + if image_exists { + tracing::info!( + "Image {} already exists in containers-storage, skipping ostree export", + &imgref.image + ); + } else { + // The image was installed from containers-storage and now only exists in ostree. + // We need to export from ostree to default containers-storage (/var/lib/containers) + tracing::info!("Image not found in containers-storage; exporting from ostree"); + // Use image_status we already obtained above (no additional unwraps needed) + let source = ImageReference { + transport: Transport::try_from(imgref.transport.as_str())?, + name: imgref.image.clone(), + }; + let target = ImageReference { + transport: Transport::ContainerStorage, + name: imgref.image.clone(), + }; + + let mut opts = ostree_ext::container::store::ExportToOCIOpts::default(); + // TODO: bridge to progress API + opts.progress_to_stdout = true; + tracing::info!( + "Exporting ostree deployment to default containers-storage: {}", + &imgref.image + ); + ostree_ext::container::store::export(repo, &source, &target, Some(opts)).await?; + } + + // Now copy from default containers-storage to bootc storage + tracing::info!( + "Copying from default containers-storage to bootc storage: {}", + &imgref.image + ); + let image_name = imgref.image.clone(); + let copy_msg = format!("Copying {} to bootc storage", &image_name); + async_task_with_spinner(©_msg, async move { + imgstore.pull_from_host_storage(&image_name).await + }) + .await?; + } else { + // For registry and other transports, check if the image already exists in + // the host's default container storage (/var/lib/containers/storage). + // If so, we can copy from there instead of pulling from the network, + // which is faster (especially after https://github.com/containers/container-libs/issues/144 + // enables reflinks between container storages). + let image_in_host = image_exists_in_host_storage(&imgref.image).await?; + + if image_in_host { + tracing::info!( + "Image {} found in host container storage; copying to bootc storage", + &imgref.image + ); + let image_name = imgref.image.clone(); + let copy_msg = format!("Copying {} to bootc storage", &image_name); + async_task_with_spinner(©_msg, async move { + imgstore.pull_from_host_storage(&image_name).await + }) + .await?; + } else { + let img_string = imgref.to_transport_image(); + let pull_msg = format!("Pulling {} to bootc storage", &img_string); + async_task_with_spinner(&pull_msg, async move { + imgstore + .pull(&img_string, crate::podstorage::PullMode::Always) + .await + }) + .await?; + } + } + + // Verify the image is now in bootc storage + let imgstore = sysroot.get_ensure_imgstore()?; + if !imgstore.exists(&imgref.image).await? { + anyhow::bail!( + "Image was pushed to bootc storage but not found: {}. \ + This may indicate a storage configuration issue.", + &imgref.image + ); + } + tracing::info!("Image verified in bootc storage: {}", &imgref.image); + + // Optionally verify we can import from containers-storage by preparing in a temp importer + // without actually importing into the main repo; this is a lightweight validation. + let containers_storage_imgref = crate::spec::ImageReference { + transport: "containers-storage".to_string(), + image: imgref.image.clone(), + signature: imgref.signature.clone(), + }; + let ostree_imgref = + ostree_ext::container::OstreeImageReference::from(containers_storage_imgref); + let _ = + ostree_ext::container::store::ImageImporter::new(repo, &ostree_imgref, Default::default()) + .await?; + + tracing::info!( + message_id = SET_UNIFIED_JOURNAL_ID, + bootc.status = "set_unified_complete", + "Unified storage set for current image. Future upgrade/switch will use it automatically." + ); + Ok(()) +} diff --git a/crates/lib/src/install.rs b/crates/lib/src/install.rs index 61aaa4d23..f2b545801 100644 --- a/crates/lib/src/install.rs +++ b/crates/lib/src/install.rs @@ -147,6 +147,15 @@ pub(crate) struct InstallTargetOpts { #[clap(long)] #[serde(default)] pub(crate) skip_fetch_check: bool, + + /// Use unified storage path to pull images (experimental) + /// + /// When enabled, this uses bootc's container storage (/usr/lib/bootc/storage) to pull + /// the image first, then imports it from there. This is the same approach used for + /// logically bound images. + #[clap(long = "experimental-unified-storage")] + #[serde(default)] + pub(crate) unified_storage_exp: bool, } #[derive(clap::Args, Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] @@ -467,6 +476,7 @@ pub(crate) struct State { pub(crate) selinux_state: SELinuxFinalState, #[allow(dead_code)] pub(crate) config_opts: InstallConfigOpts, + pub(crate) target_opts: InstallTargetOpts, pub(crate) target_imgref: ostree_container::OstreeImageReference, #[allow(dead_code)] pub(crate) prepareroot_config: HashMap, @@ -872,6 +882,7 @@ async fn install_container( state: &State, root_setup: &RootSetup, sysroot: &ostree::Sysroot, + storage: &Storage, has_ostree: bool, ) -> Result<(ostree::Deployment, InstallAleph)> { let sepolicy = state.load_policy()?; @@ -912,9 +923,26 @@ async fn install_container( let repo = &sysroot.repo(); repo.set_disable_fsync(true); - let pulled_image = match prepare_for_pull(repo, &spec_imgref, Some(&state.target_imgref)) + // Determine whether to use unified storage path. + // During install, we only use unified storage if explicitly requested. + // Auto-detection (None) is only appropriate for upgrade/switch on a running system. + let use_unified = state.target_opts.unified_storage_exp; + + let prepared = if use_unified { + tracing::info!("Using unified storage path for installation"); + crate::deploy::prepare_for_pull_unified( + repo, + &spec_imgref, + Some(&state.target_imgref), + storage, + Some(&root_setup.physical_root_path), + ) .await? - { + } else { + prepare_for_pull(repo, &spec_imgref, Some(&state.target_imgref)).await? + }; + + let pulled_image = match prepared { PreparedPullResult::AlreadyPresent(existing) => existing, PreparedPullResult::Ready(image_meta) => { check_disk_space(root_setup.physical_root.as_fd(), &image_meta, &spec_imgref)?; @@ -1475,6 +1503,7 @@ async fn prepare_install( selinux_state, source, config_opts, + target_opts, target_imgref, install_config, prepareroot_config, @@ -1529,7 +1558,7 @@ async fn install_with_sysroot( // And actually set up the container in that root, returning a deployment and // the aleph state (see below). - let (deployment, aleph) = install_container(state, rootfs, ostree, has_ostree).await?; + let (deployment, aleph) = install_container(state, rootfs, ostree, storage, has_ostree).await?; // Write the aleph data that captures the system state at the time of provisioning for aid in future debugging. aleph.write_to(&rootfs.physical_root)?; diff --git a/crates/lib/src/podstorage.rs b/crates/lib/src/podstorage.rs index eaff56572..9773d93f4 100644 --- a/crates/lib/src/podstorage.rs +++ b/crates/lib/src/podstorage.rs @@ -311,7 +311,7 @@ impl CStorage { .names .iter() .flatten() - .any(|name| !roots.contains(name.as_str())) + .all(|name| !roots.contains(name.as_str())) { garbage.push(image.id); } @@ -387,7 +387,7 @@ impl CStorage { Ok(()) } - fn subpath() -> Utf8PathBuf { + pub(crate) fn subpath() -> Utf8PathBuf { Utf8Path::new(crate::store::BOOTC_ROOT).join(SUBPATH) } } diff --git a/crates/lib/src/spec.rs b/crates/lib/src/spec.rs index bc16b2a2f..85bceb92a 100644 --- a/crates/lib/src/spec.rs +++ b/crates/lib/src/spec.rs @@ -132,6 +132,23 @@ impl ImageReference { } } } + + /// Parse the transport string into a Transport enum. + pub fn transport(&self) -> Transport { + Transport::try_from(self.transport.as_str()).unwrap_or(Transport::Registry) + } + + /// Convert to a container reference string suitable for use with container storage APIs. + /// For registry transport, returns just the image name. For other transports, prepends the transport. + pub fn to_transport_image(&self) -> String { + if self.transport() == Transport::Registry { + // For registry transport, the image name is already in the right format + self.image.clone() + } else { + // For other transports (containers-storage, oci, etc.), prepend the transport + format!("{}:{}", self.transport, self.image) + } + } } /// The status of the booted image @@ -665,4 +682,37 @@ mod tests { host.filter_to_slot(Slot::Rollback); assert_host_state(&host, None, None, Some(default_boot_entry())); } + + #[test] + fn test_to_transport_image() { + // Test registry transport (should return only the image name) + let registry_ref = ImageReference { + transport: "registry".to_string(), + image: "quay.io/example/foo:latest".to_string(), + signature: None, + }; + assert_eq!( + registry_ref.to_transport_image(), + "quay.io/example/foo:latest" + ); + + // Test containers-storage transport + let storage_ref = ImageReference { + transport: "containers-storage".to_string(), + image: "localhost/bootc".to_string(), + signature: None, + }; + assert_eq!( + storage_ref.to_transport_image(), + "containers-storage:localhost/bootc" + ); + + // Test oci transport + let oci_ref = ImageReference { + transport: "oci".to_string(), + image: "/path/to/image".to_string(), + signature: None, + }; + assert_eq!(oci_ref.to_transport_image(), "oci:/path/to/image"); + } } diff --git a/crates/lib/src/status.rs b/crates/lib/src/status.rs index f35330ef4..2b9fc613e 100644 --- a/crates/lib/src/status.rs +++ b/crates/lib/src/status.rs @@ -239,7 +239,7 @@ fn imagestatus( /// Given an OSTree deployment, parse out metadata into our spec. #[context("Reading deployment metadata")] -fn boot_entry_from_deployment( +pub(crate) fn boot_entry_from_deployment( sysroot: &SysrootLock, deployment: &ostree::Deployment, ) -> Result { diff --git a/docs/src/man/bootc-install-to-disk.8.md b/docs/src/man/bootc-install-to-disk.8.md index b470b7200..520830595 100644 --- a/docs/src/man/bootc-install-to-disk.8.md +++ b/docs/src/man/bootc-install-to-disk.8.md @@ -92,6 +92,10 @@ partition is auto-discovered. Verify the image can be fetched from the bootc image. Updates may fail when the installation host is authenticated with the registry but the pull secret is not in the bootc image +**--experimental-unified-storage** + + Use unified storage path to pull images (experimental) + **--disable-selinux** Disable SELinux in the target (installed) system diff --git a/docs/src/man/bootc-install-to-existing-root.8.md b/docs/src/man/bootc-install-to-existing-root.8.md index 416a1268e..90a446265 100644 --- a/docs/src/man/bootc-install-to-existing-root.8.md +++ b/docs/src/man/bootc-install-to-existing-root.8.md @@ -167,6 +167,10 @@ of migrating the fstab entries. See the "Injecting kernel arguments" section abo Verify the image can be fetched from the bootc image. Updates may fail when the installation host is authenticated with the registry but the pull secret is not in the bootc image +**--experimental-unified-storage** + + Use unified storage path to pull images (experimental) + **--disable-selinux** Disable SELinux in the target (installed) system diff --git a/docs/src/man/bootc-install-to-filesystem.8.md b/docs/src/man/bootc-install-to-filesystem.8.md index de8d0af9e..481b27f61 100644 --- a/docs/src/man/bootc-install-to-filesystem.8.md +++ b/docs/src/man/bootc-install-to-filesystem.8.md @@ -75,6 +75,10 @@ is currently expected to be empty by default. Verify the image can be fetched from the bootc image. Updates may fail when the installation host is authenticated with the registry but the pull secret is not in the bootc image +**--experimental-unified-storage** + + Use unified storage path to pull images (experimental) + **--disable-selinux** Disable SELinux in the target (installed) system diff --git a/tmt/plans/integration.fmf b/tmt/plans/integration.fmf index 87dfb10c3..20ab7fe91 100644 --- a/tmt/plans/integration.fmf +++ b/tmt/plans/integration.fmf @@ -130,4 +130,18 @@ execute: how: fmf test: - /tmt/tests/tests/test-29-soft-reboot-selinux-policy + +/plan-30-install-unified-flag: + summary: Test bootc install with experimental unified storage flag + discover: + how: fmf + test: + - /tmt/tests/tests/test-30-install-unified-flag + +/plan-31-switch-to-unified: + summary: Onboard to unified storage, build derived image, and switch to it + discover: + how: fmf + test: + - /tmt/tests/tests/test-31-switch-to-unified # END GENERATED PLANS diff --git a/tmt/tests/booted/test-install-unified-flag.nu b/tmt/tests/booted/test-install-unified-flag.nu new file mode 100644 index 000000000..e49a89a22 --- /dev/null +++ b/tmt/tests/booted/test-install-unified-flag.nu @@ -0,0 +1,47 @@ +# number: 30 +# extra: +# tmt: +# summary: Test bootc install with experimental unified storage flag +# duration: 30m +# +# Test bootc install with --experimental-unified-storage flag +# This test performs an actual install to a loopback device and verifies +# the unified storage path is used. + +use std assert +use tap.nu + +# Use a generic target image to test skew between the bootc binary doing +# the install and the target image +let target_image = "docker://quay.io/centos-bootc/centos-bootc:stream10" + +def main [] { + tap begin "install with experimental unified storage flag" + + # Setup filesystem - create a loopback disk image + mkdir /var/mnt + truncate -s 10G disk.img + + # Disable SELinux enforcement for the install (same as test-install-outside-container) + setenforce 0 + + # Perform the install with unified storage flag + # We use systemd-run to handle mount namespace issues + systemd-run -p MountFlags=slave -qdPG -- /bin/sh -c $" +set -xeuo pipefail +if test -d /sysroot/ostree; then mount --bind /usr/share/empty /sysroot/ostree; fi +mkdir -p /tmp/ovl/{upper,work} +mount -t overlay -olowerdir=/usr,workdir=/tmp/ovl/work,upperdir=/tmp/ovl/upper overlay /usr +# Note we do keep the other bootupd state +rm -vrf /usr/lib/bootupd/updates +# Another bootc install bug, we should not look at this in outside-of-container flows +rm -vrf /usr/lib/bootc/bound-images.d +# Install with unified storage flag to loopback disk +bootc install to-disk --disable-selinux --via-loopback --filesystem xfs --experimental-unified-storage --source-imgref ($target_image) ./disk.img +" + + # Cleanup + rm -f disk.img + + tap ok +} diff --git a/tmt/tests/booted/test-switch-to-unified.nu b/tmt/tests/booted/test-switch-to-unified.nu new file mode 100644 index 000000000..79645a288 --- /dev/null +++ b/tmt/tests/booted/test-switch-to-unified.nu @@ -0,0 +1,97 @@ +# number: 31 +# tmt: +# summary: Onboard to unified storage, build derived image, and switch to it +# duration: 30m +# +use std assert +use tap.nu + +# Multi-boot test: boot 0 onboards to unified storage and builds a derived image; +# boot 1 verifies we booted into the derived image using containers-storage + +# This code runs on *each* boot - capture status for verification +bootc status +let st = bootc status --json | from json +let booted = $st.status.booted.image + +def main [] { + match $env.TMT_REBOOT_COUNT? { + null | "0" => first_boot, + "1" => second_boot, + "2" => third_boot, + $o => { error make { msg: $"Invalid TMT_REBOOT_COUNT ($o)" } }, + } +} + +def first_boot [] { + tap begin "copy image to podman storage, switch, then onboard to unified storage" + + # Copy the currently booted image to podman storage + bootc image copy-to-storage + + # Switch to the base image using containers-storage transport + bootc switch --transport containers-storage localhost/bootc + + tmt-reboot +} + +def second_boot [] { + + # Onboard to unified storage - this pulls the booted image into bootc storage + bootc image set-unified + + # Verify bootc-owned store has the image + bootc image cmd list + podman --storage-opt=additionalimagestore=/usr/lib/bootc/storage images + + let td = mktemp -d + cd $td + + # Build a derived image with a marker file to verify we switched + # Use bootc image cmd build which builds directly into bootc storage + $"FROM localhost/bootc +RUN echo 'unified-storage-test-marker' > /usr/share/unified-storage-test.txt +" | save Dockerfile + + bootc image cmd build -t localhost/bootc-unified-derived . + + # Verify the build is in bootc storage + bootc image cmd list + + # Switch to the derived image using containers-storage transport + print "Switching to localhost/bootc-unified-derived" + bootc switch --transport containers-storage localhost/bootc-unified-derived + + tmt-reboot +} + +def third_boot [] { + tap begin "verify unified storage switch worked" + + # Verify we're booted from containers-storage transport + assert equal $booted.image.transport containers-storage + assert equal $booted.image.image localhost/bootc-unified-derived + + # Verify the marker file from our derived image exists + assert ("/usr/share/unified-storage-test.txt" | path exists) + let marker = open /usr/share/unified-storage-test.txt | str trim + assert equal $marker "unified-storage-test-marker" + + # Verify that bootc storage is accessible + print "Listing images in bootc storage:" + bootc image cmd list + + # Verify that podman can see bootc storage as additional image store + print "Testing podman access to bootc storage" + let images = podman --storage-opt=additionalimagestore=/usr/lib/bootc/storage images --format "{{.Repository}}" + print $"Images visible via podman: ($images)" + + # The derived image (localhost/bootc-unified-derived) should persist in bootc storage + # since /usr/lib/bootc/storage is a symlink to persistent storage under /sysroot. + # The key verification is that we successfully booted into the derived image, + # which we already confirmed above via transport and image name checks. + + tap ok +} + + diff --git a/tmt/tests/test-30-install-unified-flag.fmf b/tmt/tests/test-30-install-unified-flag.fmf new file mode 100644 index 000000000..ff9faa85b --- /dev/null +++ b/tmt/tests/test-30-install-unified-flag.fmf @@ -0,0 +1,3 @@ +summary: Test bootc install with experimental unified storage flag +test: nu booted/test-install-unified-flag.nu +duration: 30m diff --git a/tmt/tests/test-31-switch-to-unified.fmf b/tmt/tests/test-31-switch-to-unified.fmf new file mode 100644 index 000000000..c3a480927 --- /dev/null +++ b/tmt/tests/test-31-switch-to-unified.fmf @@ -0,0 +1,3 @@ +summary: Onboard to unified storage, build derived image, and switch to it +test: nu booted/test-switch-to-unified.nu +duration: 30m diff --git a/tmt/tests/tests.fmf b/tmt/tests/tests.fmf index b867456a4..51ba7ba2e 100644 --- a/tmt/tests/tests.fmf +++ b/tmt/tests/tests.fmf @@ -68,3 +68,13 @@ summary: Test soft reboot with SELinux policy changes duration: 30m test: nu booted/test-soft-reboot-selinux-policy.nu + +/test-30-install-unified-flag: + summary: Test bootc install with experimental unified storage flag + duration: 30m + test: nu booted/test-install-unified-flag.nu + +/test-31-switch-to-unified: + summary: Onboard to unified storage, build derived image, and switch to it + duration: 30m + test: nu booted/test-switch-to-unified.nu