diff --git a/Cargo.lock b/Cargo.lock index aba4f1e92..7cd3d3e78 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -236,6 +236,7 @@ dependencies = [ "canon-json", "cap-std-ext", "cfg-if", + "cfsctl", "chrono", "clap", "clap_mangen", @@ -469,6 +470,22 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "cfsctl" +version = "0.3.0" +source = "git+https://github.com/containers/composefs-rs?rev=2a8007f27c2252fa47f9912abbc687d723161f88#2a8007f27c2252fa47f9912abbc687d723161f88" +dependencies = [ + "anyhow", + "clap", + "composefs", + "composefs-boot", + "composefs-oci", + "env_logger 0.11.8", + "hex", + "rustix", + "tokio", +] + [[package]] name = "chrono" version = "0.4.42" @@ -559,7 +576,7 @@ checksum = "55b672471b4e9f9e95499ea597ff64941a309b2cdbffcc46f2cc5e2d971fd335" [[package]] name = "composefs" version = "0.3.0" -source = "git+https://github.com/containers/composefs-rs?rev=0f636031a1ec81cdd9e7f674909ef6b75c2642cb#0f636031a1ec81cdd9e7f674909ef6b75c2642cb" +source = "git+https://github.com/containers/composefs-rs?rev=2a8007f27c2252fa47f9912abbc687d723161f88#2a8007f27c2252fa47f9912abbc687d723161f88" dependencies = [ "anyhow", "hex", @@ -579,7 +596,7 @@ dependencies = [ [[package]] name = "composefs-boot" version = "0.3.0" -source = "git+https://github.com/containers/composefs-rs?rev=0f636031a1ec81cdd9e7f674909ef6b75c2642cb#0f636031a1ec81cdd9e7f674909ef6b75c2642cb" +source = "git+https://github.com/containers/composefs-rs?rev=2a8007f27c2252fa47f9912abbc687d723161f88#2a8007f27c2252fa47f9912abbc687d723161f88" dependencies = [ "anyhow", "composefs", @@ -592,7 +609,7 @@ dependencies = [ [[package]] name = "composefs-oci" version = "0.3.0" -source = "git+https://github.com/containers/composefs-rs?rev=0f636031a1ec81cdd9e7f674909ef6b75c2642cb#0f636031a1ec81cdd9e7f674909ef6b75c2642cb" +source = "git+https://github.com/containers/composefs-rs?rev=2a8007f27c2252fa47f9912abbc687d723161f88#2a8007f27c2252fa47f9912abbc687d723161f88" dependencies = [ "anyhow", "async-compression", @@ -903,6 +920,15 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" +[[package]] +name = "env_filter" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bf3c259d255ca70051b30e2e95b5446cdb8949ac4cd22c0d7fd634d89f568e2" +dependencies = [ + "log", +] + [[package]] name = "env_home" version = "0.1.0" @@ -919,6 +945,16 @@ dependencies = [ "regex", ] +[[package]] +name = "env_logger" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c863f0904021b108aa8b2f55046443e6b1ebde8fd4a15c399893aae4fa069f" +dependencies = [ + "env_filter", + "log", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -2012,7 +2048,7 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "588f6378e4dd99458b60ec275b4477add41ce4fa9f64dcba6f15adccb19b50d6" dependencies = [ - "env_logger", + "env_logger 0.8.4", "log", "rand 0.8.5", ] diff --git a/Cargo.toml b/Cargo.toml index ff0288f7a..9a788ab22 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,9 +41,10 @@ chrono = { version = "0.4.38", default-features = false } clap = "4.5.4" clap_mangen = { version = "0.2.20" } # Reviewers (including AI tools): This hash is duplicated as there's not a much better way to handle it right now -composefs = { git = "https://github.com/containers/composefs-rs", rev = "0f636031a1ec81cdd9e7f674909ef6b75c2642cb", package = "composefs", features = ["rhel9"] } -composefs-boot = { git = "https://github.com/containers/composefs-rs", rev = "0f636031a1ec81cdd9e7f674909ef6b75c2642cb", package = "composefs-boot" } -composefs-oci = { git = "https://github.com/containers/composefs-rs", rev = "0f636031a1ec81cdd9e7f674909ef6b75c2642cb", package = "composefs-oci" } +cfsctl = { git = "https://github.com/containers/composefs-rs", rev = "2a8007f27c2252fa47f9912abbc687d723161f88", package = "cfsctl", features = ["rhel9"] } +composefs = { git = "https://github.com/containers/composefs-rs", rev = "2a8007f27c2252fa47f9912abbc687d723161f88", package = "composefs", features = ["rhel9"] } +composefs-boot = { git = "https://github.com/containers/composefs-rs", rev = "2a8007f27c2252fa47f9912abbc687d723161f88", package = "composefs-boot" } +composefs-oci = { git = "https://github.com/containers/composefs-rs", rev = "2a8007f27c2252fa47f9912abbc687d723161f88", package = "composefs-oci" } fn-error-context = "0.2.1" hex = "0.4.3" indicatif = "0.18.0" diff --git a/crates/lib/Cargo.toml b/crates/lib/Cargo.toml index cdc278d49..8cee0968e 100644 --- a/crates/lib/Cargo.toml +++ b/crates/lib/Cargo.toml @@ -34,6 +34,7 @@ cfg-if = { workspace = true } chrono = { workspace = true, features = ["serde"] } clap = { workspace = true, features = ["derive","cargo"] } clap_mangen = { workspace = true, optional = true } +cfsctl = { workspace = true } composefs = { workspace = true } composefs-boot = { workspace = true } composefs-oci = { workspace = true } diff --git a/crates/lib/src/bootc_composefs/repo.rs b/crates/lib/src/bootc_composefs/repo.rs index c3f478169..35fd47b55 100644 --- a/crates/lib/src/bootc_composefs/repo.rs +++ b/crates/lib/src/bootc_composefs/repo.rs @@ -3,10 +3,7 @@ use std::sync::Arc; use anyhow::{Context, Result}; -use ostree_ext::composefs::{ - fsverity::{FsVerityHashValue, Sha512HashValue}, - util::Sha256Digest, -}; +use ostree_ext::composefs::fsverity::{FsVerityHashValue, Sha512HashValue}; use ostree_ext::composefs_boot::{bootloader::BootEntry as ComposefsBootEntry, BootOps}; use ostree_ext::composefs_oci::{ image::create_filesystem as create_composefs_filesystem, pull as composefs_oci_pull, @@ -26,7 +23,7 @@ pub(crate) fn open_composefs_repo(rootfs_dir: &Dir) -> Result Result<(Sha256Digest, impl FsVerityHashValue)> { +) -> Result<(String, impl FsVerityHashValue)> { let rootfs_dir = &root_setup.physical_root; rootfs_dir @@ -94,11 +91,11 @@ pub(crate) async fn pull_composefs_repo( .await .context("Pulling composefs repo")?; - tracing::info!("ID: {}, Verity: {}", hex::encode(id), verity.to_hex()); + tracing::info!("ID: {id}, Verity: {}", verity.to_hex()); let repo = open_composefs_repo(&rootfs_dir)?; let mut fs: crate::store::ComposefsFilesystem = - create_composefs_filesystem(&repo, &hex::encode(id), None) + create_composefs_filesystem(&repo, &id, None) .context("Failed to create composefs filesystem")?; let entries = fs.transform_for_boot(&repo)?; diff --git a/crates/lib/src/bootc_composefs/update.rs b/crates/lib/src/bootc_composefs/update.rs index eebfd3faa..f36e11931 100644 --- a/crates/lib/src/bootc_composefs/update.rs +++ b/crates/lib/src/bootc_composefs/update.rs @@ -1,6 +1,5 @@ use anyhow::{Context, Result}; use camino::Utf8PathBuf; -use composefs::util::{parse_sha256, Sha256Digest}; use fn_error_context::context; use ostree_ext::oci_spec::image::{ImageConfiguration, ImageManifest}; @@ -17,12 +16,6 @@ use crate::{ store::{BootedComposefs, ComposefsRepository, Storage}, }; -#[context("Getting SHA256 Digest for {id}")] -pub fn str_to_sha256digest(id: &str) -> Result { - let id = id.strip_prefix("sha256:").unwrap_or(id); - Ok(parse_sha256(&id)?) -} - /// Checks if a container image has been pulled to the local composefs repository. /// /// This function verifies whether the specified container image exists in the local @@ -50,10 +43,9 @@ async fn is_image_pulled( let (manifest, config) = get_container_manifest_and_config(&imgref_repr).await?; let img_digest = manifest.config().digest().digest(); - let img_sha256 = str_to_sha256digest(&img_digest)?; - // check_stream is expensive to run, but probably a good idea - let container_pulled = repo.check_stream(&img_sha256).context("Checking stream")?; + // NB: add deep checking? + let container_pulled = repo.has_stream(&img_digest).context("Checking stream")?; Ok((container_pulled.is_some(), manifest, config)) } @@ -122,8 +114,6 @@ pub(crate) async fn upgrade_composefs( // TODO(Johan-Liebert1): If we have the previous, i.e. the current manifest with us then we can replace the // following with [`ostree_container::ManifestDiff::new`] which will be much cleaner for (idx, diff_id) in config.rootfs().diff_ids().iter().enumerate() { - let diff_id = str_to_sha256digest(diff_id)?; - // we could use `check_stream` here but that will most probably take forever as it // usually takes ~3s to verify one single layer let have_layer = repo.has_stream(&diff_id)?; diff --git a/crates/lib/src/cfsctl.rs b/crates/lib/src/cfsctl.rs deleted file mode 100644 index 63a005d58..000000000 --- a/crates/lib/src/cfsctl.rs +++ /dev/null @@ -1,372 +0,0 @@ -use std::{ - ffi::OsString, - fs::{create_dir_all, File}, - io::BufWriter, - path::{Path, PathBuf}, - sync::Arc, -}; - -use anyhow::{Context, Result}; -use camino::Utf8PathBuf; -use clap::{Parser, Subcommand}; - -use rustix::fs::CWD; - -use composefs_boot::{write_boot, BootOps}; - -use composefs::{ - dumpfile, - fsverity::{FsVerityHashValue, Sha512HashValue}, - repository::Repository, -}; - -/// cfsctl -#[derive(Debug, Parser)] -#[clap(name = "cfsctl", version)] -pub struct App { - #[clap(long, group = "repopath")] - repo: Option, - #[clap(long, group = "repopath")] - user: bool, - #[clap(long, group = "repopath")] - system: bool, - - /// Sets the repository to insecure before running any operation and - /// prepend '?' to the composefs kernel command line when writing - /// boot entry. - #[clap(long)] - insecure: bool, - - #[clap(subcommand)] - cmd: Command, -} - -#[derive(Debug, Subcommand)] -enum OciCommand { - /// Stores a tar file as a splitstream in the repository. - ImportLayer { - sha256: String, - name: Option, - }, - /// Lists the contents of a tar stream - LsLayer { - /// the name of the stream - name: String, - }, - Dump { - config_name: String, - config_verity: Option, - }, - Pull { - image: String, - name: Option, - }, - ComputeId { - config_name: String, - config_verity: Option, - #[clap(long)] - bootable: bool, - }, - CreateImage { - config_name: String, - config_verity: Option, - #[clap(long)] - bootable: bool, - #[clap(long)] - image_name: Option, - }, - Seal { - config_name: String, - config_verity: Option, - }, - Mount { - name: String, - mountpoint: String, - }, - PrepareBoot { - config_name: String, - config_verity: Option, - #[clap(long, default_value = "/boot")] - bootdir: PathBuf, - #[clap(long)] - entry_id: Option, - #[clap(long)] - cmdline: Vec, - }, -} - -#[derive(Debug, Subcommand)] -enum Command { - /// Take a transaction lock on the repository. - /// This prevents garbage collection from occurring. - Transaction, - /// Reconstitutes a split stream and writes it to stdout - Cat { - /// the name of the stream to cat, either a sha256 digest or prefixed with 'ref/' - name: String, - }, - /// Perform garbage collection - GC, - /// Imports a composefs image (unsafe!) - ImportImage { - reference: String, - }, - /// Commands for dealing with OCI layers - Oci { - #[clap(subcommand)] - cmd: OciCommand, - }, - /// Mounts a composefs, possibly enforcing fsverity of the image - Mount { - /// the name of the image to mount, either a sha256 digest or prefixed with 'ref/' - name: String, - /// the mountpoint - mountpoint: String, - }, - CreateImage { - path: PathBuf, - #[clap(long)] - bootable: bool, - #[clap(long)] - stat_root: bool, - image_name: Option, - }, - ComputeId { - path: PathBuf, - /// Write the dumpfile to the provided target - #[clap(long)] - write_dumpfile_to: Option, - #[clap(long)] - bootable: bool, - #[clap(long)] - stat_root: bool, - }, - CreateDumpfile { - path: PathBuf, - #[clap(long)] - bootable: bool, - #[clap(long)] - stat_root: bool, - }, - ImageObjects { - name: String, - }, -} - -fn verity_opt(opt: &Option) -> Result> { - Ok(opt.as_ref().map(FsVerityHashValue::from_hex).transpose()?) -} - -pub(crate) async fn run_from_iter(args: I) -> Result<()> -where - I: IntoIterator, - I::Item: Into + Clone, -{ - let args = App::parse_from( - std::iter::once(OsString::from("cfs")).chain(args.into_iter().map(Into::into)), - ); - - let repo = if let Some(path) = &args.repo { - let mut r = Repository::open_path(CWD, path)?; - r.set_insecure(args.insecure); - Arc::new(r) - } else if args.user { - let mut r = Repository::open_user()?; - r.set_insecure(args.insecure); - Arc::new(r) - } else { - if args.insecure { - anyhow::bail!("Cannot override insecure state for system repo"); - } - let system_store = crate::cli::get_storage().await?; - system_store.get_ensure_composefs()? - }; - let repo = &repo; - - match args.cmd { - Command::Transaction => { - // just wait for ^C - loop { - std::thread::park(); - } - } - Command::Cat { name } => { - repo.merge_splitstream(&name, None, &mut std::io::stdout())?; - } - Command::ImportImage { reference } => { - let image_id = repo.import_image(&reference, &mut std::io::stdin())?; - println!("{}", image_id.to_id()); - } - Command::Oci { cmd: oci_cmd } => match oci_cmd { - OciCommand::ImportLayer { name, sha256 } => { - let object_id = composefs_oci::import_layer( - &repo, - &composefs::util::parse_sha256(sha256)?, - name.as_deref(), - &mut std::io::stdin(), - )?; - println!("{}", object_id.to_id()); - } - OciCommand::LsLayer { name } => { - composefs_oci::ls_layer(&repo, &name)?; - } - OciCommand::Dump { - ref config_name, - ref config_verity, - } => { - let verity = verity_opt(config_verity)?; - let mut fs = - composefs_oci::image::create_filesystem(&repo, config_name, verity.as_ref())?; - fs.print_dumpfile()?; - } - OciCommand::ComputeId { - ref config_name, - ref config_verity, - bootable, - } => { - let verity = verity_opt(config_verity)?; - let mut fs = - composefs_oci::image::create_filesystem(&repo, config_name, verity.as_ref())?; - if bootable { - fs.transform_for_boot(&repo)?; - } - let id = fs.compute_image_id(); - println!("{}", id.to_hex()); - } - OciCommand::CreateImage { - ref config_name, - ref config_verity, - bootable, - ref image_name, - } => { - let verity = verity_opt(config_verity)?; - let mut fs = - composefs_oci::image::create_filesystem(&repo, config_name, verity.as_ref())?; - if bootable { - fs.transform_for_boot(&repo)?; - } - let image_id = fs.commit_image(&repo, image_name.as_deref())?; - println!("{}", image_id.to_id()); - } - OciCommand::Pull { ref image, name } => { - let (sha256, verity) = - composefs_oci::pull(&repo, image, name.as_deref(), None).await?; - - println!("sha256 {}", hex::encode(sha256)); - println!("verity {}", verity.to_hex()); - } - OciCommand::Seal { - ref config_name, - ref config_verity, - } => { - let verity = verity_opt(config_verity)?; - let (sha256, verity) = composefs_oci::seal(&repo, config_name, verity.as_ref())?; - println!("sha256 {}", hex::encode(sha256)); - println!("verity {}", verity.to_id()); - } - OciCommand::Mount { - ref name, - ref mountpoint, - } => { - composefs_oci::mount(&repo, name, mountpoint, None)?; - } - OciCommand::PrepareBoot { - ref config_name, - ref config_verity, - ref bootdir, - ref entry_id, - ref cmdline, - } => { - let verity = verity_opt(config_verity)?; - let mut fs = - composefs_oci::image::create_filesystem(&repo, config_name, verity.as_ref())?; - let entries = fs.transform_for_boot(&repo)?; - let id = fs.commit_image(&repo, None)?; - - let Some(entry) = entries.into_iter().next() else { - anyhow::bail!("No boot entries!"); - }; - - let cmdline_refs: Vec<&str> = cmdline.iter().map(String::as_str).collect(); - write_boot::write_boot_simple( - &repo, - entry, - &id, - args.insecure, - bootdir, - None, - entry_id.as_deref(), - &cmdline_refs, - )?; - - let state = args - .repo - .as_ref() - .map(|p: &PathBuf| p.parent().unwrap_or(p)) - .unwrap_or(Path::new("/sysroot")) - .join("state/deploy") - .join(id.to_hex()); - - create_dir_all(state.join("var"))?; - create_dir_all(state.join("etc/upper"))?; - create_dir_all(state.join("etc/work"))?; - } - }, - Command::ComputeId { - ref path, - write_dumpfile_to, - bootable, - stat_root, - } => { - let mut fs = composefs::fs::read_filesystem(CWD, path, Some(&repo), stat_root)?; - if bootable { - fs.transform_for_boot(&repo)?; - } - let id = fs.compute_image_id(); - println!("{}", id.to_hex()); - if let Some(path) = write_dumpfile_to.as_deref() { - let mut w = File::create(path) - .with_context(|| format!("Opening {path}")) - .map(BufWriter::new)?; - dumpfile::write_dumpfile(&mut w, &fs).context("Writing dumpfile")?; - } - } - Command::CreateImage { - ref path, - bootable, - stat_root, - ref image_name, - } => { - let mut fs = composefs::fs::read_filesystem(CWD, path, Some(&repo), stat_root)?; - if bootable { - fs.transform_for_boot(&repo)?; - } - let id = fs.commit_image(&repo, image_name.as_deref())?; - println!("{}", id.to_id()); - } - Command::CreateDumpfile { - ref path, - bootable, - stat_root, - } => { - let mut fs = composefs::fs::read_filesystem(CWD, path, Some(&repo), stat_root)?; - if bootable { - fs.transform_for_boot(&repo)?; - } - fs.print_dumpfile()?; - } - Command::Mount { name, mountpoint } => { - repo.mount_at(&name, &mountpoint)?; - } - Command::ImageObjects { name } => { - let objects = repo.objects_for_image(&name)?; - for object in objects { - println!("{}", object.to_id()); - } - } - Command::GC => { - repo.gc()?; - } - } - Ok(()) -} diff --git a/crates/lib/src/cli.rs b/crates/lib/src/cli.rs index 23bae71d9..7d6a04433 100644 --- a/crates/lib/src/cli.rs +++ b/crates/lib/src/cli.rs @@ -1521,10 +1521,12 @@ async fn run_from_opt(opt: Opt) -> Result<()> { let storage = get_storage().await?; let cfs = storage.get_ensure_composefs()?; let testdata = b"some test data"; - let testdata_digest = openssl::sha::sha256(testdata); - let mut w = SplitStreamWriter::new(&cfs, None, Some(testdata_digest)); + let testdata_digest = hex::encode(openssl::sha::sha256(testdata)); + let mut w = SplitStreamWriter::new(&cfs, 0); w.write_inline(testdata); - let object = cfs.write_stream(w, Some("testobject"))?.to_hex(); + let object = cfs + .write_stream(w, &testdata_digest, Some("testobject"))? + .to_hex(); assert_eq!(object, "5d94ceb0b2bb3a78237e0a74bc030a262239ab5f47754a5eb2e42941056b64cb21035d64a8f7c2f156e34b820802fa51884de2b1f7dc3a41b9878fc543cd9b07"); Ok(()) } @@ -1545,7 +1547,14 @@ async fn run_from_opt(opt: Opt) -> Result<()> { Ok(()) } }, - InternalsOpts::Cfs { args } => crate::cfsctl::run_from_iter(args.iter()).await, + InternalsOpts::Cfs { args } => { + let args = cfsctl::App::parse_from( + std::iter::once(OsString::from("cfs")).chain(args.into_iter().map(Into::into)), + ); + + cfsctl::main::(args).await?; + Ok(()) + } InternalsOpts::Reboot => crate::reboot::reboot(), InternalsOpts::Fsck => { let storage = &get_storage().await?; diff --git a/crates/lib/src/install.rs b/crates/lib/src/install.rs index 12e65c599..a86916159 100644 --- a/crates/lib/src/install.rs +++ b/crates/lib/src/install.rs @@ -1676,9 +1676,9 @@ async fn install_to_filesystem_impl( // Load a fd for the mounted target physical root let (id, verity) = initialize_composefs_repository(state, rootfs).await?; - tracing::info!("id: {}, verity: {}", hex::encode(id), verity.to_hex()); + tracing::info!("id: {id}, verity: {}", verity.to_hex()); - setup_composefs_boot(rootfs, state, &hex::encode(id))?; + setup_composefs_boot(rootfs, state, &id)?; } else { ostree_install(state, rootfs, cleanup).await?; } diff --git a/crates/lib/src/lib.rs b/crates/lib/src/lib.rs index 905f167ad..8d6ac40e9 100644 --- a/crates/lib/src/lib.rs +++ b/crates/lib/src/lib.rs @@ -8,7 +8,6 @@ mod bootc_composefs; pub(crate) mod bootc_kargs; mod bootloader; mod boundimage; -mod cfsctl; pub mod cli; mod composefs_consts; mod containerenv; diff --git a/docs/src/man/bootc.8.md b/docs/src/man/bootc.8.md index 3c22aacb4..dd9ee7d18 100644 --- a/docs/src/man/bootc.8.md +++ b/docs/src/man/bootc.8.md @@ -34,7 +34,6 @@ pulled and `bootc upgrade`. | **bootc install** | Install the running container to a target | | **bootc container** | Operations which can be executed as part of a container build | | **bootc composefs-finalize-staged** | | -| **bootc config-diff** | Diff current /etc configuration versus default |