From b5bafa52533e96a92fbc32393a40f52fadbb0722 Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Fri, 16 Feb 2024 10:29:37 -0500 Subject: [PATCH 1/7] osbuild: move manifests into a subdirectory I'm planning to break them up a little and having them in their own directory will be useful for that. --- src/cmd-buildextend-metal | 6 +++--- src/{ => osbuild-manifests}/coreos.osbuild.aarch64.mpp.yaml | 0 src/{ => osbuild-manifests}/coreos.osbuild.ppc64le.mpp.yaml | 0 src/{ => osbuild-manifests}/coreos.osbuild.s390x.mpp.yaml | 0 src/{ => osbuild-manifests}/coreos.osbuild.x86_64.mpp.yaml | 0 5 files changed, 3 insertions(+), 3 deletions(-) rename src/{ => osbuild-manifests}/coreos.osbuild.aarch64.mpp.yaml (100%) rename src/{ => osbuild-manifests}/coreos.osbuild.ppc64le.mpp.yaml (100%) rename src/{ => osbuild-manifests}/coreos.osbuild.s390x.mpp.yaml (100%) rename src/{ => osbuild-manifests}/coreos.osbuild.x86_64.mpp.yaml (100%) diff --git a/src/cmd-buildextend-metal b/src/cmd-buildextend-metal index c696e74015..b824c238e0 100755 --- a/src/cmd-buildextend-metal +++ b/src/cmd-buildextend-metal @@ -290,9 +290,9 @@ if [ "${OSBUILD_SUPPORTED:-}" != "" ] && [ "${COSA_USE_OSBUILD:-}" != "" ]; then # so for those we'll set `snapshot=on` so that each will get their own disk image. # This is OK because we don't checkpoint (cache) any of those stages. [ "${image_type}" == "qemu" ] && snapshot="off" || snapshot="on" - runvm_with_cache_snapshot "$snapshot" -- /usr/lib/coreos-assembler/runvm-osbuild \ - --config "${image_for_disk_json}" \ - --mpp "/usr/lib/coreos-assembler/coreos.osbuild.${basearch}.mpp.yaml" \ + runvm_with_cache_snapshot "$snapshot" -- /usr/lib/coreos-assembler/runvm-osbuild \ + --config "${image_for_disk_json}" \ + --mpp "/usr/lib/coreos-assembler/osbuild-manifests/coreos.osbuild.${basearch}.mpp.yaml" \ --filepath "${path}.tmp" else runvm "${qemu_args[@]}" -- \ diff --git a/src/coreos.osbuild.aarch64.mpp.yaml b/src/osbuild-manifests/coreos.osbuild.aarch64.mpp.yaml similarity index 100% rename from src/coreos.osbuild.aarch64.mpp.yaml rename to src/osbuild-manifests/coreos.osbuild.aarch64.mpp.yaml diff --git a/src/coreos.osbuild.ppc64le.mpp.yaml b/src/osbuild-manifests/coreos.osbuild.ppc64le.mpp.yaml similarity index 100% rename from src/coreos.osbuild.ppc64le.mpp.yaml rename to src/osbuild-manifests/coreos.osbuild.ppc64le.mpp.yaml diff --git a/src/coreos.osbuild.s390x.mpp.yaml b/src/osbuild-manifests/coreos.osbuild.s390x.mpp.yaml similarity index 100% rename from src/coreos.osbuild.s390x.mpp.yaml rename to src/osbuild-manifests/coreos.osbuild.s390x.mpp.yaml diff --git a/src/coreos.osbuild.x86_64.mpp.yaml b/src/osbuild-manifests/coreos.osbuild.x86_64.mpp.yaml similarity index 100% rename from src/coreos.osbuild.x86_64.mpp.yaml rename to src/osbuild-manifests/coreos.osbuild.x86_64.mpp.yaml From 915e0bc808e52bd1c56b048d95374cc6a04dc7ee Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Fri, 16 Feb 2024 11:42:24 -0500 Subject: [PATCH 2/7] runvm-osbuild: make ostree_repo and ostree_ref optional inputs They aren't used in the deploy-via-container path so let's make them not required if the caller happens to not include them. --- src/runvm-osbuild | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/runvm-osbuild b/src/runvm-osbuild index 1a651d3e18..49f8d567cc 100755 --- a/src/runvm-osbuild +++ b/src/runvm-osbuild @@ -46,8 +46,8 @@ done # Get the base filename of the desired file output path filename=$(basename "$filepath") -ostree_repo=$(getconfig "ostree-repo") -ostree_ref=$(getconfig "ostree-ref") +ostree_repo=$(getconfig_def "ostree-repo" "") +ostree_ref=$(getconfig_def "ostree-ref" "") osname=$(getconfig "osname") platform=$(getconfig "image-type") deploy_via_container=$(getconfig_def "deploy-via-container" "") From 77cd7a80eaf2229378c5680c80bf32c476d3f470 Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Fri, 16 Feb 2024 13:10:40 -0500 Subject: [PATCH 3/7] osbuild: add hyperv platform artifact definition file Currently there is nothing in platforms.yaml for hyperv so it can be shared across the two architectures we build it for today. We don't currently use this but will in the future. --- .../coreos.osbuild.aarch64.mpp.yaml | 2 + .../coreos.osbuild.x86_64.mpp.yaml | 2 + .../platform.hyperv.ipp.yaml | 57 +++++++++++++++++++ 3 files changed, 61 insertions(+) create mode 100644 src/osbuild-manifests/platform.hyperv.ipp.yaml diff --git a/src/osbuild-manifests/coreos.osbuild.aarch64.mpp.yaml b/src/osbuild-manifests/coreos.osbuild.aarch64.mpp.yaml index 8d652580df..351bf2fb81 100644 --- a/src/osbuild-manifests/coreos.osbuild.aarch64.mpp.yaml +++ b/src/osbuild-manifests/coreos.osbuild.aarch64.mpp.yaml @@ -676,3 +676,5 @@ pipelines: type: qcow2 compression: false compat: '1.1' + - mpp-import-pipelines: + path: platform.hyperv.ipp.yaml diff --git a/src/osbuild-manifests/coreos.osbuild.x86_64.mpp.yaml b/src/osbuild-manifests/coreos.osbuild.x86_64.mpp.yaml index ff34d0953d..e5246a790e 100644 --- a/src/osbuild-manifests/coreos.osbuild.x86_64.mpp.yaml +++ b/src/osbuild-manifests/coreos.osbuild.x86_64.mpp.yaml @@ -682,3 +682,5 @@ pipelines: type: qcow2 compression: false compat: '1.1' + - mpp-import-pipelines: + path: platform.hyperv.ipp.yaml diff --git a/src/osbuild-manifests/platform.hyperv.ipp.yaml b/src/osbuild-manifests/platform.hyperv.ipp.yaml new file mode 100644 index 0000000000..803da7d3c9 --- /dev/null +++ b/src/osbuild-manifests/platform.hyperv.ipp.yaml @@ -0,0 +1,57 @@ +# This file isn't yet being used today but holds an OSBuild image +# definition for the hyperv platform. +version: '2' +pipelines: + - name: raw-hyperv-image + stages: + - type: org.osbuild.copy + inputs: + tree: + type: org.osbuild.tree + origin: org.osbuild.pipeline + references: + - name:raw-image + options: + paths: + - from: input://tree/disk.img + to: tree:///disk.img + # Increase the size to the cloud image size + - type: org.osbuild.truncate + options: + filename: disk.img + size: + mpp-format-string: "{cloud_image_size_mb * 1024 * 1024}" + - type: org.osbuild.kernel-cmdline.bls-append + options: + bootpath: mount:/// + kernel_opts: + - ignition.platform.id=hyperv + devices: + boot: + type: org.osbuild.loopback + options: + filename: disk.img + start: + mpp-format-int: '{image.layout[''boot''].start}' + size: + mpp-format-int: '{image.layout[''boot''].size}' + mounts: + - name: boot + type: org.osbuild.ext4 + source: boot + target: / + - name: hyperv + stages: + - type: org.osbuild.qemu + inputs: + image: + type: org.osbuild.files + origin: org.osbuild.pipeline + references: + name:raw-hyperv-image: + file: disk.img + options: + filename: + mpp-format-string: '{filename}' + format: + type: vhdx From 4a4929049f977b373b8cae8a77f1ac40d3043695 Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Fri, 16 Feb 2024 13:17:52 -0500 Subject: [PATCH 4/7] osbuild: add applehv platform artifact definition file Currently there is nothing in platforms.yaml for applehv so it can be shared across the two architectures we build it for today. We don't currently use this but will in the future. --- .../coreos.osbuild.aarch64.mpp.yaml | 2 + .../coreos.osbuild.x86_64.mpp.yaml | 2 + .../platform.applehv.ipp.yaml | 56 +++++++++++++++++++ 3 files changed, 60 insertions(+) create mode 100644 src/osbuild-manifests/platform.applehv.ipp.yaml diff --git a/src/osbuild-manifests/coreos.osbuild.aarch64.mpp.yaml b/src/osbuild-manifests/coreos.osbuild.aarch64.mpp.yaml index 351bf2fb81..b1ff131f2c 100644 --- a/src/osbuild-manifests/coreos.osbuild.aarch64.mpp.yaml +++ b/src/osbuild-manifests/coreos.osbuild.aarch64.mpp.yaml @@ -676,5 +676,7 @@ pipelines: type: qcow2 compression: false compat: '1.1' + - mpp-import-pipelines: + path: platform.applehv.ipp.yaml - mpp-import-pipelines: path: platform.hyperv.ipp.yaml diff --git a/src/osbuild-manifests/coreos.osbuild.x86_64.mpp.yaml b/src/osbuild-manifests/coreos.osbuild.x86_64.mpp.yaml index e5246a790e..5da4f59b4a 100644 --- a/src/osbuild-manifests/coreos.osbuild.x86_64.mpp.yaml +++ b/src/osbuild-manifests/coreos.osbuild.x86_64.mpp.yaml @@ -682,5 +682,7 @@ pipelines: type: qcow2 compression: false compat: '1.1' + - mpp-import-pipelines: + path: platform.applehv.ipp.yaml - mpp-import-pipelines: path: platform.hyperv.ipp.yaml diff --git a/src/osbuild-manifests/platform.applehv.ipp.yaml b/src/osbuild-manifests/platform.applehv.ipp.yaml new file mode 100644 index 0000000000..c584cb2b2f --- /dev/null +++ b/src/osbuild-manifests/platform.applehv.ipp.yaml @@ -0,0 +1,56 @@ +# This file isn't yet being used today but holds an OSBuild image +# definition for the applehv platform. +version: '2' +pipelines: + - name: raw-applehv-image + stages: + - type: org.osbuild.copy + inputs: + tree: + type: org.osbuild.tree + origin: org.osbuild.pipeline + references: + - name:raw-image + options: + paths: + - from: input://tree/disk.img + to: tree:///disk.img + # Increase the size to the cloud image size + - type: org.osbuild.truncate + options: + filename: disk.img + size: + mpp-format-string: "{cloud_image_size_mb * 1024 * 1024}" + - type: org.osbuild.kernel-cmdline.bls-append + options: + bootpath: mount:/// + kernel_opts: + - ignition.platform.id=applehv + devices: + boot: + type: org.osbuild.loopback + options: + filename: disk.img + start: + mpp-format-int: '{image.layout[''boot''].start}' + size: + mpp-format-int: '{image.layout[''boot''].size}' + mounts: + - name: boot + type: org.osbuild.ext4 + source: boot + target: / + - name: applehv + stages: + - type: org.osbuild.copy + inputs: + tree: + type: org.osbuild.tree + origin: org.osbuild.pipeline + references: + - name:raw-applehv-image + options: + paths: + - from: input://tree/disk.img + to: + mpp-format-string: 'tree:///{filename}' From 276ae81ed422adad7558c31653e43e6ce2793791 Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Fri, 16 Feb 2024 13:19:11 -0500 Subject: [PATCH 5/7] osbuild: add a stub manifest for qemu This isn't used today but will help with some automation I am building so that the process won't break in the future when we switch to defining the `qemu` pieces in this file. --- src/osbuild-manifests/platform.qemu.ipp.yaml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 src/osbuild-manifests/platform.qemu.ipp.yaml diff --git a/src/osbuild-manifests/platform.qemu.ipp.yaml b/src/osbuild-manifests/platform.qemu.ipp.yaml new file mode 100644 index 0000000000..c3e21203d9 --- /dev/null +++ b/src/osbuild-manifests/platform.qemu.ipp.yaml @@ -0,0 +1,2 @@ +# This file isn't used today but will hold the OSBuild image +# defintion for the qemu platform in the future. From 62517c2b9082687a58c9b1613112bcf5ff3a6fc7 Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Fri, 16 Feb 2024 13:25:36 -0500 Subject: [PATCH 6/7] src/runvm-osbuild: make loop-control conditional Sometimes I iterate on this inside the supermin VM and the script fails on the second and subsequent runs. Let's just make it not make /dev/loop-control if it already exists. --- src/runvm-osbuild | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runvm-osbuild b/src/runvm-osbuild index 49f8d567cc..bfc40b7ce2 100755 --- a/src/runvm-osbuild +++ b/src/runvm-osbuild @@ -63,7 +63,7 @@ if [ -n "${deploy_via_container}" ]; then fi # Since it doesn't exist create loop-control -mknod /dev/loop-control c 10 237 +[ ! -e /dev/loop-control ] && mknod /dev/loop-control c 10 237 # Put the store and the output dir on the cache. At the end we'll mv # out the created artifact from the output dir to the place it's supposed From f51455bc3f357002aabb99fa693f426e9356dc2e Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Fri, 16 Feb 2024 13:51:33 -0500 Subject: [PATCH 7/7] src/runvm-osbuild: use a temp file for the processed json We were using /processed.json because we run this inside a supermin VM which is essentially an entire root filesystem that's throwaway, but there's a new case where we might want to run it outside of a supermin VM for the podman folks so let's not pollute the root dir and use a temporary directory instead. --- src/runvm-osbuild | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/runvm-osbuild b/src/runvm-osbuild index bfc40b7ce2..785b3e4807 100755 --- a/src/runvm-osbuild +++ b/src/runvm-osbuild @@ -71,6 +71,8 @@ fi outdir=cache/osbuild/out storedir=cache/osbuild/store +processed_json=$(mktemp -t osbuild-XXXX.json) + # Run through the preprocessor # Note: don't quote the size arguements since they are numbers, not strings osbuild-mpp \ @@ -82,8 +84,7 @@ osbuild-mpp \ -D container_imgref=\""${container_imgref}"\" \ -D metal_image_size_mb="${metal_image_size_mb}" \ -D cloud_image_size_mb="${cloud_image_size_mb}" \ - "${mppyaml}" \ - /processed.json + "${mppyaml}" "${processed_json}" # Build the image osbuild \ @@ -92,9 +93,11 @@ osbuild \ --cache-max-size 9GiB \ --checkpoint tree \ --checkpoint raw-image \ - --export "$platform" /processed.json + --export "$platform" "${processed_json}" # Copy it out to the specified location. Use mv here so we remove it # from the cache qcow2 so we don't cache it. mv "${outdir}/${platform}/${filename}" "${filepath}" + +rm -f "${processed_json}"