diff --git a/conf.py b/conf.py index 85a818d58..c3a016e16 100644 --- a/conf.py +++ b/conf.py @@ -26,6 +26,7 @@ opa = importlib.import_module("opa.versions") spark_k8s = importlib.import_module("spark-k8s.versions") stackable_base = importlib.import_module("stackable-base.versions") +stackable_devel = importlib.import_module("stackable-devel.versions") superset = importlib.import_module("superset.versions") trino_cli = importlib.import_module("trino-cli.versions") trino = importlib.import_module("trino.versions") @@ -55,6 +56,7 @@ {"name": "opa", "versions": opa.versions}, {"name": "spark-k8s", "versions": spark_k8s.versions}, {"name": "stackable-base", "versions": stackable_base.versions}, + {"name": "stackable-devel", "versions": stackable_devel.versions}, {"name": "superset", "versions": superset.versions}, {"name": "trino-cli", "versions": trino_cli.versions}, {"name": "trino", "versions": trino.versions}, diff --git a/druid/Dockerfile b/druid/Dockerfile index c4c928f7e..a45d6b02a 100644 --- a/druid/Dockerfile +++ b/druid/Dockerfile @@ -23,10 +23,7 @@ microdnf update # This requirement is documented in docs/development/build.md and version 5.1 or later is required. # UBI 9 ships with 5.4.x so that should be fine # -# patch: Required for the apply-patches.sh script -microdnf install \ - python-pyyaml \ - patch +microdnf install python-pyyaml microdnf clean all rm -rf /var/cache/yum @@ -35,8 +32,7 @@ EOF USER ${STACKABLE_USER_UID} WORKDIR /stackable -COPY --chown=${STACKABLE_USER_UID}:0 druid/stackable/patches/apply_patches.sh /stackable/apache-druid-${PRODUCT}-src/patches/apply_patches.sh -COPY --chown=${STACKABLE_USER_UID}:0 druid/stackable/patches/${PRODUCT} /stackable/apache-druid-${PRODUCT}-src/patches/${PRODUCT} +COPY --chown=${STACKABLE_USER_UID}:0 druid/stackable/patches/${PRODUCT} /stackable/src/druid/stackable/patches/${PRODUCT} # Cache mounts are owned by root by default # We need to explicitly give the uid to use which is hardcoded to "1000" in stackable-base @@ -50,9 +46,7 @@ RUN --mount=type=cache,id=maven-${PRODUCT},uid=${STACKABLE_USER_UID},target=/sta --mount=type=cache,id=npm-${PRODUCT},uid=${STACKABLE_USER_UID},target=/stackable/.npm \ --mount=type=cache,id=cache-${PRODUCT},uid=${STACKABLE_USER_UID},target=/stackable/.cache \ < /stackable/bin/hbck2 chmod +x /stackable/bin/hbck2 @@ -246,20 +241,13 @@ ARG STACKABLE_USER_UID # This can be used to speed up builds when disk space is of no concern. ARG DELETE_CACHES="true" -COPY --chown=${STACKABLE_USER_UID}:0 hbase/phoenix/stackable/patches /stackable/patches -COPY --chown=${STACKABLE_USER_UID}:0 hbase/stackable/patches/apply_patches.sh /stackable/patches +COPY --chown=${STACKABLE_USER_UID}:0 hbase/phoenix/stackable/patches/${PHOENIX} /stackable/src/phoenix/stackable/patches/${PHOENIX} USER ${STACKABLE_USER_UID} WORKDIR /stackable RUN --mount=type=cache,id=maven-phoenix-${PHOENIX},uid=${STACKABLE_USER_UID},target=/stackable/.m2/repository < { +struct ProductVersionContext { pv: ProductVersion, - images_repo_root: &'a Path, + images_repo_root: PathBuf, } -impl ProductVersionContext<'_> { +impl ProductVersionContext { fn load_config(&self) -> Result { let path = &self.config_path(); tracing::info!( @@ -107,6 +103,10 @@ impl ProductVersionContext<'_> { struct Opts { #[clap(subcommand)] cmd: Cmd, + + /// Specify a custom root directory for the images repository + #[clap(long)] + images_repo_root: Option, } #[derive(clap::Parser)] @@ -165,7 +165,7 @@ enum Cmd { pv: ProductVersion, }, - /// Shwos the images repository root + /// Shows the images repository root ImagesDir, } @@ -263,8 +263,16 @@ fn main() -> Result<()> { .context(ConfigureGitLoggingSnafu)?; let opts = ::parse(); - let images_repo = repo::discover_images_repo(".").context(FindImagesRepoSnafu)?; - let images_repo_root = images_repo.workdir().context(NoImagesRepoWorkdirSnafu)?; + let images_repo_root = match opts.images_repo_root { + Some(path) => path, + None => { + let images_repo = repo::discover_images_repo(".").context(FindImagesRepoSnafu)?; + images_repo + .workdir() + .context(NoImagesRepoWorkdirSnafu)? + .to_owned() + } + }; match opts.cmd { Cmd::Checkout { pv, base_only } => { let ctx = ProductVersionContext { diff --git a/spark-k8s/Dockerfile b/spark-k8s/Dockerfile index 4a1276c9d..4f20dc1bb 100644 --- a/spark-k8s/Dockerfile +++ b/spark-k8s/Dockerfile @@ -7,43 +7,17 @@ FROM stackable/image/hadoop AS hadoop-builder # hbase-builder: Provides HBase libraries FROM stackable/image/hbase AS hbase-builder -# spark-source-builder: Download the Spark source code into -# /stackable/spark and apply the patches +# spark-source-builder: Clone the Spark source code and apply patches FROM stackable/image/java-devel AS spark-source-builder ARG PRODUCT ARG STACKABLE_USER_UID -RUN <>> Build spark diff --git a/spark-k8s/hbase-connectors/apply_patches.sh b/spark-k8s/hbase-connectors/apply_patches.sh deleted file mode 100755 index 833b3e9c7..000000000 --- a/spark-k8s/hbase-connectors/apply_patches.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash - -# Enable error handling and unset variable checking -set -eu -set -o pipefail - -# Check if $1 (VERSION) is provided -if [ -z "${1-}" ]; then - echo "Please provide a value for VERSION as the first argument." - exit 1 -fi - -VERSION="$1" -PATCH_DIR="patches/$VERSION" - -# Check if version-specific patches directory exists -if [ ! -d "$PATCH_DIR" ]; then - echo "Patches directory '$PATCH_DIR' does not exist." - exit 1 -fi - -# Create an array to hold the patches in sorted order -declare -a patch_files=() - -echo "Applying patches from ${PATCH_DIR}" now - -# Read the patch files into the array -while IFS= read -r -d $'\0' file; do - patch_files+=("$file") -done < <(find "$PATCH_DIR" -name "*.patch" -print0 | sort -zV) - -echo "Found ${#patch_files[@]} patches, applying now" - -# Iterate through sorted patch files -for patch_file in "${patch_files[@]}"; do - echo "Applying $patch_file" - # We can not use Git here, as we are not within a Git repo - patch --directory "." --strip=1 < "$patch_file" || { - echo "Failed to apply $patch_file" - exit 1 - } -done - -echo "All patches applied successfully." diff --git a/spark-k8s/stackable/patches/apply_patches.sh b/spark-k8s/stackable/patches/apply_patches.sh deleted file mode 100755 index 833b3e9c7..000000000 --- a/spark-k8s/stackable/patches/apply_patches.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash - -# Enable error handling and unset variable checking -set -eu -set -o pipefail - -# Check if $1 (VERSION) is provided -if [ -z "${1-}" ]; then - echo "Please provide a value for VERSION as the first argument." - exit 1 -fi - -VERSION="$1" -PATCH_DIR="patches/$VERSION" - -# Check if version-specific patches directory exists -if [ ! -d "$PATCH_DIR" ]; then - echo "Patches directory '$PATCH_DIR' does not exist." - exit 1 -fi - -# Create an array to hold the patches in sorted order -declare -a patch_files=() - -echo "Applying patches from ${PATCH_DIR}" now - -# Read the patch files into the array -while IFS= read -r -d $'\0' file; do - patch_files+=("$file") -done < <(find "$PATCH_DIR" -name "*.patch" -print0 | sort -zV) - -echo "Found ${#patch_files[@]} patches, applying now" - -# Iterate through sorted patch files -for patch_file in "${patch_files[@]}"; do - echo "Applying $patch_file" - # We can not use Git here, as we are not within a Git repo - patch --directory "." --strip=1 < "$patch_file" || { - echo "Failed to apply $patch_file" - exit 1 - } -done - -echo "All patches applied successfully." diff --git a/stackable-base/Dockerfile b/stackable-base/Dockerfile index f38d5b614..b40b09788 100644 --- a/stackable-base/Dockerfile +++ b/stackable-base/Dockerfile @@ -1,61 +1,29 @@ # syntax=docker/dockerfile:1.10.0@sha256:865e5dd094beca432e8c0a1d5e1c465db5f998dca4e439981029b3b81fb39ed5 # check=error=true -# Find the latest version at https://catalog.redhat.com/software/containers/ubi9/ubi-minimal/615bd9b4075b022acc111bf5?container-tabs=gti -# IMPORTANT: Make sure to use the "Manifest List Digest" that references the images for multiple architectures -# rather than just the "Image Digest" that references the image for the selected architecture. -FROM registry.access.redhat.com/ubi9/ubi-minimal@sha256:c0e70387664f30cd9cf2795b547e4a9a51002c44a4a86aa9335ab030134bf392 AS product-utils-builder +FROM stackable/image/stackable-devel AS config-utils # Find the latest version here: https://github.com/stackabletech/config-utils/tags # renovate: datasource=github-tags packageName=stackabletech/config-utils ENV CONFIG_UTILS_VERSION=0.2.0 -# Find the latest version here: https://github.com/stackabletech/containerdebug/tags -# renovate: datasource=github-tags packageName=stackabletech/containerdebug -ENV CONTAINERDEBUG_VERSION=0.1.1 -# This SHOULD be kept in sync with operator-templating and other tools to reduce build times -# Find the latest version here: https://doc.rust-lang.org/stable/releases.html -# renovate: datasource=github-releases packageName=rust-lang/rust -ENV RUST_DEFAULT_TOOLCHAIN_VERSION=1.84.1 -# Find the latest version here: https://crates.io/crates/cargo-cyclonedx -# renovate: datasource=crate packageName=cargo-cyclonedx -ENV CARGO_CYCLONEDX_CRATE_VERSION=0.5.7 -# Find the latest version here: https://crates.io/crates/cargo-auditable -# renovate: datasource=crate packageName=cargo-auditable -ENV CARGO_AUDITABLE_CRATE_VERSION=0.6.6 - -RUN <