feat: move patch apply logic to patchable

conf.py

@@ -26,6 +26,7 @@
 opa = importlib.import_module("opa.versions")
 spark_k8s = importlib.import_module("spark-k8s.versions")
 stackable_base = importlib.import_module("stackable-base.versions")
+stackable_devel = importlib.import_module("stackable-devel.versions")
 superset = importlib.import_module("superset.versions")
 trino_cli = importlib.import_module("trino-cli.versions")
 trino = importlib.import_module("trino.versions")
@@ -54,6 +55,7 @@
     {"name": "opa", "versions": opa.versions},
     {"name": "spark-k8s", "versions": spark_k8s.versions},
     {"name": "stackable-base", "versions": stackable_base.versions},
+    {"name": "stackable-devel", "versions": stackable_devel.versions},
     {"name": "superset", "versions": superset.versions},
     {"name": "trino-cli", "versions": trino_cli.versions},
     {"name": "trino", "versions": trino.versions},

druid/Dockerfile

@@ -23,10 +23,8 @@ microdnf update
 #   This requirement is documented in docs/development/build.md and version 5.1 or later is required.
 #   UBI 9 ships with 5.4.x so that should be fine
 #
-# patch: Required for the apply-patches.sh script
 microdnf install \
-python-pyyaml \
-patch
+python-pyyaml
 
 microdnf clean all
 rm -rf /var/cache/yum
@@ -35,8 +33,7 @@ EOF
 USER ${STACKABLE_USER_UID}
 WORKDIR /stackable
 
-COPY --chown=stackable:0 druid/stackable/patches/apply_patches.sh /stackable/apache-druid-${PRODUCT}-src/patches/apply_patches.sh
-COPY --chown=stackable:0 druid/stackable/patches/${PRODUCT} /stackable/apache-druid-${PRODUCT}-src/patches/${PRODUCT}
+COPY --chown=${STACKABLE_USER_UID}:0 druid/stackable/patches/${PRODUCT} /stackable/src/druid/stackable/patches/${PRODUCT}
 
 # Cache mounts are owned by root by default
 # We need to explicitly give the uid to use which is hardcoded to "1000" in stackable-base
@@ -50,10 +47,7 @@ RUN --mount=type=cache,id=maven-${PRODUCT},uid=${STACKABLE_USER_UID},target=/sta
     --mount=type=cache,id=npm-${PRODUCT},uid=${STACKABLE_USER_UID},target=/stackable/.npm \
     --mount=type=cache,id=cache-${PRODUCT},uid=${STACKABLE_USER_UID},target=/stackable/.cache \
     <<EOF
-curl "https://repo.stackable.tech/repository/packages/druid/apache-druid-${PRODUCT}-src.tar.gz" | tar -xzC .
-cd apache-druid-${PRODUCT}-src
-./patches/apply_patches.sh ${PRODUCT}
-
+cd "$(/stackable/patchable --images-repo-root=src checkout druid ${PRODUCT})" && \
 mvn --batch-mode --no-transfer-progress clean install -Pdist,stackable-bundle-contrib-exts -DskipTests -Dmaven.javadoc.skip=true
 mv distribution/target/apache-druid-${PRODUCT}-bin/apache-druid-${PRODUCT} /stackable/
 mv distribution/target/bom.json /stackable/apache-druid-${PRODUCT}/apache-druid-${PRODUCT}.cdx.json

java-devel/Dockerfile

@@ -2,10 +2,10 @@
 # check=error=true
 
 #
-# Base imaege for builder stages
+# Base image for builder stages in Java based products
 #
 
-FROM stackable/image/stackable-base
+FROM stackable/image/stackable-devel
 
 ARG PRODUCT
 

java-devel/versions.py

@@ -1,26 +1,26 @@
 versions = [
     {
         "product": "8",
-        "stackable-base": "1.0.0",
+        "stackable-devel": "1.0.0",
     },
     {
         "product": "11",
-        "stackable-base": "1.0.0",
+        "stackable-devel": "1.0.0",
     },
     {
         "product": "17",
-        "stackable-base": "1.0.0",
+        "stackable-devel": "1.0.0",
     },
     {
         "product": "21",
-        "stackable-base": "1.0.0",
+        "stackable-devel": "1.0.0",
     },
     {
         "product": "22",
-        "stackable-base": "1.0.0",
+        "stackable-devel": "1.0.0",
     },
     {
         "product": "23",
-        "stackable-base": "1.0.0",
+        "stackable-devel": "1.0.0",
     },
 ]

rust/patchable/src/main.rs

@@ -5,11 +5,7 @@ mod repo;
 mod utils;
 
 use core::str;
-use std::{
-    fs::File,
-    io::Write,
-    path::{Path, PathBuf},
-};
+use std::{fs::File, io::Write, path::PathBuf};
 
 use git2::{Oid, Repository};
 use serde::{Deserialize, Serialize};
@@ -35,12 +31,12 @@ struct ProductVersionConfig {
     base: Oid,
 }
 
-struct ProductVersionContext<'a> {
+struct ProductVersionContext {
     pv: ProductVersion,
-    images_repo_root: &'a Path,
+    images_repo_root: PathBuf,
 }
 
-impl ProductVersionContext<'_> {
+impl ProductVersionContext {
     fn load_config(&self) -> Result<ProductVersionConfig> {
         let path = &self.config_path();
         tracing::info!(
@@ -107,6 +103,10 @@ impl ProductVersionContext<'_> {
 struct Opts {
     #[clap(subcommand)]
     cmd: Cmd,
+
+    /// Specify a custom root directory for the images repository
+    #[clap(long)]
+    images_repo_root: Option<PathBuf>,
 }
 
 #[derive(clap::Parser)]
@@ -165,7 +165,7 @@ enum Cmd {
         pv: ProductVersion,
     },
 
-    /// Shwos the images repository root
+    /// Shows the images repository root
     ImagesDir,
 }
 
@@ -263,8 +263,16 @@ fn main() -> Result<()> {
     .context(ConfigureGitLoggingSnafu)?;
 
     let opts = <Opts as clap::Parser>::parse();
-    let images_repo = repo::discover_images_repo(".").context(FindImagesRepoSnafu)?;
-    let images_repo_root = images_repo.workdir().context(NoImagesRepoWorkdirSnafu)?;
+    let images_repo_root = match opts.images_repo_root {
+        Some(path) => path,
+        None => {
+            let images_repo = repo::discover_images_repo(".").context(FindImagesRepoSnafu)?;
+            images_repo
+                .workdir()
+                .context(NoImagesRepoWorkdirSnafu)?
+                .to_owned()
+        }
+    };
     match opts.cmd {
         Cmd::Checkout { pv, base_only } => {
             let ctx = ProductVersionContext {

stackable-base/Dockerfile

@@ -1,61 +1,29 @@
 # syntax=docker/dockerfile:1.10.0@sha256:865e5dd094beca432e8c0a1d5e1c465db5f998dca4e439981029b3b81fb39ed5
 # check=error=true
 
-# Find the latest version at https://catalog.redhat.com/software/containers/ubi9/ubi-minimal/615bd9b4075b022acc111bf5?container-tabs=gti
-# IMPORTANT: Make sure to use the "Manifest List Digest" that references the images for multiple architectures
-# rather than just the "Image Digest" that references the image for the selected architecture.
-FROM registry.access.redhat.com/ubi9/ubi-minimal@sha256:c0e70387664f30cd9cf2795b547e4a9a51002c44a4a86aa9335ab030134bf392 AS product-utils-builder
+FROM stackable/image/stackable-devel AS config-utils
 
 # Find the latest version here: https://github.com/stackabletech/config-utils/tags
 # renovate: datasource=github-tags packageName=stackabletech/config-utils
 ENV CONFIG_UTILS_VERSION=0.2.0
-# Find the latest version here: https://github.com/stackabletech/containerdebug/tags
-# renovate: datasource=github-tags packageName=stackabletech/containerdebug
-ENV CONTAINERDEBUG_VERSION=0.1.1
-# This SHOULD be kept in sync with operator-templating and other tools to reduce build times
-# Find the latest version here: https://doc.rust-lang.org/stable/releases.html
-# renovate: datasource=github-releases packageName=rust-lang/rust
-ENV RUST_DEFAULT_TOOLCHAIN_VERSION=1.84.1
-# Find the latest version here: https://crates.io/crates/cargo-cyclonedx
-# renovate: datasource=crate packageName=cargo-cyclonedx
-ENV CARGO_CYCLONEDX_CRATE_VERSION=0.5.7
-# Find the latest version here: https://crates.io/crates/cargo-auditable
-# renovate: datasource=crate packageName=cargo-auditable
-ENV CARGO_AUDITABLE_CRATE_VERSION=0.6.6
-
-RUN <<EOF
-microdnf update --assumeyes
-
-# git: Needed to fetch source
-# gcc: Needed for compilation
-microdnf --assumeyes install \
-  gcc \
-  git
-microdnf clean all
-rm -rf /var/cache/yum
-
-# WARNING (@NickLarsenNZ): We should pin the rustup version
-curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain "$RUST_DEFAULT_TOOLCHAIN_VERSION"
-. "$HOME/.cargo/env" && cargo --quiet install cargo-cyclonedx@"$CARGO_CYCLONEDX_CRATE_VERSION" cargo-auditable@"$CARGO_AUDITABLE_CRATE_VERSION"
-EOF
-
-FROM product-utils-builder AS config-utils
 
 RUN <<EOF
 git clone --depth 1 --branch "${CONFIG_UTILS_VERSION}" https://github.com/stackabletech/config-utils
 cd ./config-utils
 . "$HOME/.cargo/env"
-rustup toolchain install
 cargo auditable --quiet build --release && cargo cyclonedx --all --spec-version 1.5 --describe binaries
 EOF
 
-FROM product-utils-builder AS containerdebug
+FROM stackable/image/stackable-devel AS containerdebug
+
+# Find the latest version here: https://github.com/stackabletech/containerdebug/tags
+# renovate: datasource=github-tags packageName=stackabletech/containerdebug
+ENV CONTAINERDEBUG_VERSION=0.1.1
 
 RUN <<EOF
 git clone --depth 1 --branch "${CONTAINERDEBUG_VERSION}" https://github.com/stackabletech/containerdebug
 cd ./containerdebug
 . "$HOME/.cargo/env"
-rustup toolchain install
 cargo auditable --quiet build --release && cargo cyclonedx --all --spec-version 1.5 --describe binaries
 EOF
 

stackable-base/versions.py

@@ -1,5 +1,6 @@
 versions = [
     {
         "product": "1.0.0",
+        "stackable-devel": "1.0.0",
     },
 ]

stackable-devel/Dockerfile

@@ -0,0 +1,104 @@
+# syntax=docker/dockerfile:1.10.0@sha256:865e5dd094beca432e8c0a1d5e1c465db5f998dca4e439981029b3b81fb39ed5
+# check=error=true
+
+#
+# Base image for builder stages
+#
+
+# Find the latest version at https://catalog.redhat.com/software/containers/ubi9/ubi-minimal/615bd9b4075b022acc111bf5?container-tabs=gti
+# IMPORTANT: Make sure to use the "Manifest List Digest" that references the images for multiple architectures
+# rather than just the "Image Digest" that references the image for the selected architecture.
+FROM registry.access.redhat.com/ubi9/ubi-minimal@sha256:c0e70387664f30cd9cf2795b547e4a9a51002c44a4a86aa9335ab030134bf392
+
+# intentionally unused
+ARG PRODUCT
+ARG STACKABLE_USER_UID
+ARG STACKABLE_USER_GID
+ARG STACKABLE_USER_NAME
+
+# Sets the default shell to Bash with strict error handling and robust pipeline processing.
+# "-e": Exits immediately if a command exits with a non-zero status
+# "-u": Treats unset variables as an error, preventing unexpected behavior from undefined variables.
+# "-o pipefail": Causes a pipeline to return the exit status of the last command in the pipe that failed, ensuring errors in any part of a pipeline are not ignored.
+# "-c": Allows the execution of commands passed as a string
+# This is automatically inherited in all other Dockerfiles that use this unless it is overwritten
+SHELL ["/bin/bash", "-euo", "pipefail", "-c"]
+
+# We configure microdnf to not install weak dependencies in this file
+# Not doing this caused the content of images to become unpredictable because
+# based on which packages get updated by `microdnf update` new weak dependencies
+# might be installed that were not present earlier (the ubi base image doesn't
+# seem to install weak dependencies)
+# This also affects the packages that are installed in our Dockerfiles (java as prime
+# example).
+# https://github.com/stackabletech/docker-images/pull/533
+COPY stackable-base/stackable/dnf.conf /etc/dnf/dnf.conf
+
+# This SHOULD be kept in sync with operator-templating and other tools to reduce build times
+# Find the latest version here: https://doc.rust-lang.org/stable/releases.html
+# renovate: datasource=github-releases packageName=rust-lang/rust
+ENV RUST_DEFAULT_TOOLCHAIN_VERSION=1.84.1
+# Find the latest version here: https://crates.io/crates/cargo-cyclonedx
+# renovate: datasource=crate packageName=cargo-cyclonedx
+ENV CARGO_CYCLONEDX_CRATE_VERSION=0.5.7
+# Find the latest version here: https://crates.io/crates/cargo-auditable
+# renovate: datasource=crate packageName=cargo-auditable
+ENV CARGO_AUDITABLE_CRATE_VERSION=0.6.6
+
+RUN <<EOF
+microdnf update
+
+# git: Needed to fetch source
+# gcc: Needed for compilation
+microdnf install \
+  gcc \
+  git
+
+###
+### Add Stackable user and group
+###
+# Added only temporarily to create the user and group, removed again below
+microdnf install shadow-utils
+groupadd --gid ${STACKABLE_USER_GID} --system ${STACKABLE_USER_NAME}
+
+# The --no-log-init is required to work around a bug/problem in Go/Docker when very large UIDs are used
+# See https://github.com/moby/moby/issues/5419#issuecomment-41478290 for more context
+# Making this a system user prevents a mail dir from being created, expiry of passwords etc. but it will warn:
+#   useradd warning: stackable's uid 1000 is greater than SYS_UID_MAX 999
+# We can safely ignore this warning, to get rid of the warning we could change /etc/login.defs but that does not seem worth it
+# We'll leave the home directory hardcoded to /stackable because I don't want to deal with which chars might be valid and which might not in user name vs. directory
+useradd \
+  --no-log-init \
+  --gid ${STACKABLE_USER_GID} \
+  --uid ${STACKABLE_USER_UID} \
+  --system \
+  --create-home \
+  --home-dir /stackable \
+   ${STACKABLE_USER_NAME}
+microdnf remove shadow-utils
+
+microdnf clean all
+rm -rf /var/cache/yum
+
+# WARNING (@NickLarsenNZ): We should pin the rustup version
+curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain "$RUST_DEFAULT_TOOLCHAIN_VERSION"
+. "$HOME/.cargo/env" && cargo --quiet install cargo-cyclonedx@"$CARGO_CYCLONEDX_CRATE_VERSION" cargo-auditable@"$CARGO_AUDITABLE_CRATE_VERSION" && rustup toolchain install
+EOF
+
+
+# **patchable**
+# Tool for patch management
+# Copy source code of patchable
+COPY rust/patchable/ /patchable/rust/patchable
+# Copy workspace files
+COPY Cargo.* /patchable/
+
+RUN <<EOF
+microdnf update
+microdnf install openssl-devel pkg-config
+cd /patchable
+. "$HOME/.cargo/env"
+cargo auditable --quiet build --release && cargo cyclonedx --all --spec-version 1.5 --describe binaries
+mv /patchable/target/release/patchable /stackable/patchable
+microdnf clean all
+EOF

stackable-devel/versions.py

@@ -0,0 +1,5 @@
+versions = [
+    {
+        "product": "1.0.0",
+    },
+]

zookeeper/Dockerfile

@@ -10,17 +10,17 @@ ARG PRODUCT
 ARG JMX_EXPORTER
 ARG STACKABLE_USER_UID
 
-# Copy patches and JMX config into the builder
-COPY --chown=${STACKABLE_USER_UID}:0 zookeeper/stackable /stackable
+# Copy patches into the builder
+COPY --chown=${STACKABLE_USER_UID}:0 zookeeper/stackable/patches/${PRODUCT} /stackable/src/zookeeper/stackable/patches/${PRODUCT}
+# Copy JMX config into the builder
+COPY --chown=${STACKABLE_USER_UID}:0 zookeeper/stackable/jmx /stackable/jmx
 
 USER ${STACKABLE_USER_UID}
 WORKDIR /stackable
 
 # Download ZooKeeper sources from our own repo
-RUN curl "https://repo.stackable.tech/repository/packages/zookeeper/apache-zookeeper-${PRODUCT}.tar.gz" | tar -xzC . && \
-    # Apply any required patches
-    patches/apply_patches.sh ${PRODUCT} && \
-    cd /stackable/apache-zookeeper-${PRODUCT}/ && \
+RUN BUILD_SRC_DIR="$(/stackable/patchable --images-repo-root=src checkout zookeeper ${PRODUCT})" && \
+    cd "${BUILD_SRC_DIR}" && \
     # Exclude the `zookeeper-client` submodule, this is not needed and has c parts
     # that created all kinds of issues for the build container
     mvn --batch-mode --no-transfer-progress -pl "!zookeeper-client/zookeeper-client-c" clean install checkstyle:check spotbugs:check -DskipTests -Pfull-build && \
@@ -29,7 +29,7 @@ RUN curl "https://repo.stackable.tech/repository/packages/zookeeper/apache-zooke
     # Unpack the archive which contains the build artifacts from above. Remove some
     # unused files to shrink the final image size.
     tar xvzf /stackable/apache-zookeeper-${PRODUCT}-bin.tar.gz && \
-    mv /stackable/apache-zookeeper-${PRODUCT}/zookeeper-assembly/target/bom.json /stackable/apache-zookeeper-${PRODUCT}-bin/apache-zookeeper-${PRODUCT}.cdx.json && \
+    mv "${BUILD_SRC_DIR}/zookeeper-assembly/target/bom.json" /stackable/apache-zookeeper-${PRODUCT}-bin/apache-zookeeper-${PRODUCT}.cdx.json && \
     rm -rf /stackable/apache-zookeeper-${PRODUCT}-bin/docs && \
     rm /stackable/apache-zookeeper-${PRODUCT}-bin/README_packaging.md && \
     # Download the JMX exporter jar from our own repo