get-ocp-repo.sh: drastically simplify

jlebon · jlebon · commit b8fbcd9da439 · 2025-03-30T22:05:32.000-04:00
Now that (1) we've reworked the layered node image build to only enable
the repos it needs, and (2) we've simplified the CentOS Stream GPG keys,
we can delete all of the complex logic in this repo. It basically just
boils down to curl'ing down all the repo files we may need to build the
various artifacts that use this script.
diff --git a/Containerfile b/Containerfile
@@ -40,7 +40,7 @@ find /usr -name '*.pyc' -exec mv {} {}.bak \;
 
 # fetch repos from in-cluster mirrors if we're running in OpenShift CI
 if [ "${OPENSHIFT_CI}" != 0 ]; then
-  /run/src/ci/get-ocp-repo.sh --ocp-layer /run/src/packages-openshift.yaml --output-dir /etc/yum.repos.d
+  /run/src/ci/get-ocp-repo.sh /etc/yum.repos.d/ocp.repo
 fi
 
 # XXX: patch cri-o spec to use tmpfiles
@@ -52,9 +52,9 @@ source /etc/os-release
 rpm-ostree experimental compose treefile-apply \
   --var id=$ID /run/src/packages-openshift.yaml
 
-# do any cleanups necessary to undo what `get-ocp-repo.sh` did
+# cleanup the repo file we injected
 if [ "${OPENSHIFT_CI}" != 0 ]; then
-  /run/src/ci/get-ocp-repo.sh --output-dir /etc/yum.repos.d --cleanup
+  rm /etc/yum.repos.d/ocp.repo
 fi
 
 find /usr -name '*.pyc.bak' -exec sh -c 'mv $1 ${1%.bak}' _ {} \;
diff --git a/ci/get-ocp-repo.sh b/ci/get-ocp-repo.sh
@@ -1,205 +1,19 @@
 #!/bin/bash
 set -euo pipefail
 
-# This script is used when running within the OpenShift CI clusters to fetch
-# the RHEL and OCP yum repo files from an in-cluster service that mirrors the
-# content. It's called from three places:
-# - prow-entrypoint.sh: CI tests that build & and test different variants
-# - extensions/Dockerfile: when building the extensions container in OpenShift CI
-# - Containerfile: when building the node image in CI
-
-print_usage_and_exit() {
-    cat 1>&2 <<'EOF'
-Usage: $0 <MODE> [OPTIONS]
-
-  Fetch mirrored RHEL/OCP yum repo files from OpenShift CI's in-cluster service.
-  The following modes are supported:
-
-  --cosa-workdir PATH      Get RHEL and OCP versions from manifests in cosa workdir
-  --ocp-layer    MANIFEST  Get RHEL version from /usr/lib/os-release and OCP version from manifest
-
-  The following options are supported
-
-  --output-dir   PATH      Directory to which to output ocp.repo file
-EOF
-    exit 1
-}
-
-info() {
-    echo "INFO:" "$@" >&2
-}
-
-cleanup_repos() {
-    # if we had installed the packages and created symlinks, remove it
-    if rpm -q centos-release-cloud; then
-        dnf remove -y centos-release-{cloud,nfv,virt}-common
-        find "/usr/share/distribution-gpg-keys/centos" -type l -exec rm -f {} \;
-        echo "Removed all symbolic links and packages installed for scos"
-    fi
-    # remove ocp.repo file
-    if [ -n "$ocp_manifest" ]; then
-        if [ -z "$output_dir" ]; then
-            output_dir=$(dirname "$ocp_manifest")
-        fi
-    else
-        if [ -z "$output_dir" ]; then
-            output_dir="$cosa_workdir/src/config"
-        fi
-    fi
-    rm "$output_dir/ocp.repo"
-    echo "Removed repo file $output_dir/ocp.repo"
-}
-
-create_gpg_keys() {
-    # Check if centos-stream-release is installed and centos-release-cloud is not
-    # enablerepo added in case the repo is disabled (when building extensions)
-    if rpm -q centos-stream-release && ! rpm -q centos-release-cloud; then
-        dnf install -y centos-release-{cloud,nfv,virt}-common --enablerepo extras-common
-    fi
-
-    # Create directory for CentOS distribution GPG keys
-    mkdir -p /usr/share/distribution-gpg-keys/centos
-    # Create symbolic links for GPG keys
-    if [ ! -e "/usr/share/distribution-gpg-keys/centos/RPM-GPG-KEY-CentOS-Official" ]; then
-        ln -s /etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial /usr/share/distribution-gpg-keys/centos/RPM-GPG-KEY-CentOS-Official
-        ln -s {/etc/pki/rpm-gpg,/usr/share/distribution-gpg-keys/centos}/RPM-GPG-KEY-CentOS-SIG-Cloud
-        ln -s {/etc/pki/rpm-gpg,/usr/share/distribution-gpg-keys/centos}/RPM-GPG-KEY-CentOS-SIG-Extras-SHA512
-        ln -s {/etc/pki/rpm-gpg,/usr/share/distribution-gpg-keys/centos}/RPM-GPG-KEY-CentOS-SIG-NFV
-        ln -s {/etc/pki/rpm-gpg,/usr/share/distribution-gpg-keys/centos}/RPM-GPG-KEY-CentOS-SIG-Virtualization
-    fi
-}
-
-cosa_workdir=
-ocp_manifest=
-output_dir=
-rc=0
-options=$(getopt --options h --longoptions help,cosa-workdir:,ocp-layer:,output-dir:,cleanup -- "$@") || rc=$?
-[ $rc -eq 0 ] || print_usage_and_exit
-eval set -- "$options"
-while [ $# -ne 0 ]; do
-    case "$1" in
-        -h | --help) print_usage_and_exit;;
-        --cosa-workdir) cosa_workdir=$2; shift;;
-        --ocp-layer) ocp_manifest=$2; shift;;
-        --output-dir) output_dir=$2; shift;;
-        --cleanup) cleanup_repos; exit 0;;
-        --) break;;
-        *) echo "$0: invalid argument: $1" >&2; exit 1;;
-    esac
-    shift
-done
-
-if [ -n "$ocp_manifest" ]; then
-    # --ocp-layer path
-    ocp_version=$(rpm-ostree compose tree --print-only "$ocp_manifest" | jq -r '.metadata.ocp_version')
-    ocp_version=${ocp_version//./-}
-    info "Got OpenShift version $ocp_version from $ocp_manifest"
-    # osname is used lower down, so set it
-    osname=$(source /usr/lib/os-release; if [ $ID == centos ]; then echo scos; fi)
-
-    if [ -z "$output_dir" ]; then
-        output_dir=$(dirname "$ocp_manifest")
-    fi
-
-    # get rhel version corresponding to the release so we can get the
-    # correct OpenShift rpms from those for scos. These packages are not
-    # available in CentOS Stream
-    if [ "$osname" = scos ]; then
-        workdir=$(dirname "$ocp_manifest")
-        manifest="$workdir/manifest.yaml"
-        json=$(rpm-ostree compose tree --print-only "$manifest")
-        version=$(jq -r '.["automatic-version-prefix"]' <<< "$json")
-        rhel_version=$(cut -f2 -d. <<< "$version")
-        info "Got RHEL version $rhel_version from rhel manifest for scos"
-    else
-        rhel_version=$(source /usr/lib/os-release; echo ${VERSION_ID//./})
-        info "Got RHEL version $rhel_version from /usr/lib/os-release"
-    fi
-else
-    [ -n "$cosa_workdir" ]
-    # --cosa-workdir path
-
-    # the OCP version always comes from packages-openshift.yaml
-    ocp_version=$(rpm-ostree compose tree --print-only "$cosa_workdir/src/config/packages-openshift.yaml" | jq -r '.metadata.ocp_version')
-    ocp_version=${ocp_version//./-}
-    info "Got OpenShift version $ocp_version from packages-openshift.yaml"
-
-    # the RHEL version comes from the target manifest
-
-    # first, make sure we're looking at the right manifest
-    manifest="$cosa_workdir/src/config/manifest.yaml"
-    if [ -f "$cosa_workdir/src/config.json" ]; then
-        variant="$(jq --raw-output '."coreos-assembler.config-variant"' 'src/config.json')"
-        manifest="$cosa_workdir/src/config/manifest-${variant}.yaml"
-    fi
-
-    # flatten manifest and query a couple of fields
-    json=$(rpm-ostree compose tree --print-only "$manifest")
-    osname=$(jq -r '.metadata.name' <<< "$json")
-    is_ocp_variant=$(jq '.packages | contains(["cri-o"])' <<< "$json")
-
-    if [ "$osname" = scos ] && [ "$is_ocp_variant" = false ]; then
-        # this is the pure SCOS case; we don't need any additional repos at all
-        info "Building pure SCOS variant. Exiting..."
-        exit 0
-    elif [ "$osname" = scos ]; then
-        # We still need the OCP repos for now unfortunately because not
-        # everything is in the Stream repo. For the RHEL version, just use the
-        # default variant's one.
-        json=$(rpm-ostree compose tree --print-only "$cosa_workdir/src/config/manifest.yaml")
-    fi
-    version=$(jq -r '.["automatic-version-prefix"]' <<< "$json")
-    if [ "$is_ocp_variant" = true ]; then
-        # RHEL version is second field
-        info "Building OCP variant"
-        rhel_version=$(cut -f2 -d. <<< "$version")
-    else
-        # RHEL version is first and second field
-        info "Building pure variant"
-        rhel_version=$(cut -f1-2 -d. <<< "$version")
-        rhel_version=${rhel_version//./}
-    fi
-    info "Got RHEL version $rhel_version from automatic-version-prefix value $version"
-
-    if [ -z "$output_dir" ]; then
-        output_dir="$cosa_workdir/src/config"
-    fi
-fi
-
-mkdir -p "$output_dir"
-repo_path="$output_dir/ocp.repo"
-
-set -x
-curl --fail -L "http://base-${ocp_version}-rhel${rhel_version}.ocp.svc.cluster.local" -o "$repo_path"
-set +x
-
-if [ "${rhel_version}" = 96 ]; then
-    # XXX: also currently also add 9.4 repos for crun-wasm when building extensions
+urls=(
+    # theoretically that's the only one we need
+    "http://base-4-19-rhel96.ocp.svc.cluster.local"
+    # XXX: but also currently add 9.4 repos for crun-wasm when building extensions
     # https://github.com/openshift/os/issues/1680
     # https://github.com/openshift/os/pull/1682
     # https://issues.redhat.com/browse/COS-3075
-    curl --fail -L http://base-4-19-rhel94.ocp.svc.cluster.local >> "$repo_path"
-fi
+    "http://base-4-19-rhel94.ocp.svc.cluster.local"
+)
 
-# If we're building the SCOS OKD variant, then strip away all the RHEL repos and just keep the plashet.
-# Temporary workaround until we have all packages for SCOS in CentOS Stream.
-if [ "$osname" = scos ]; then
-    info "Neutering RHEL repos for SCOS"
-    awk '/server-ose/,/^$/' "$repo_path" > "$repo_path.tmp"
-    # only pull in certain Openshift packages as the rest come from the c9s repo
-    sed -i '/^baseurl = /a includepkgs=openshift-* ose-aws-ecr-* ose-azure-acr-* ose-gcp-gcr-*' "$repo_path.tmp"
-    # add the contents of the CentOS Stream repo
-    workdir="$cosa_workdir/src/config"
-    if [ -n "$ocp_manifest" ]; then
-        workdir=$(dirname "$ocp_manifest")
-    fi
-    # pull in the mirror repo as well in case there are newer versions in the composes
-    # and we require older versions - this happens because we build the node images async
-    # and the composes move fast.
-    cat "$workdir/c9s.repo" >> "$repo_path.tmp"
-    cat "$workdir/c9s-mirror.repo" >> "$repo_path.tmp"
-    mv "$repo_path.tmp" "$repo_path"
-    create_gpg_keys
-fi
+dest=$1; shift
 
-cat "$repo_path"
+rm -f "$dest"
+for url in ${urls[$@]}; do
+    curl --fail -L "$url" >> "$dest"
+done
diff --git a/ci/prow-entrypoint.sh b/ci/prow-entrypoint.sh
@@ -60,7 +60,7 @@ cosa_init() {
 
 # Initialize the .repo files
 prepare_repos() {
-    src/config/ci/get-ocp-repo.sh --cosa-workdir .
+    src/config/ci/get-ocp-repo.sh src/config/ocp.repo
 }
 
 # Do a cosa build & cosa build-extensions only.
diff --git a/extensions-ocp-rhel-9.6.yaml b/extensions-ocp-rhel-9.6.yaml
@@ -13,7 +13,6 @@ extensions:
       # XXX: temporarily add rhel-9.4-appstream for crun-wasm
       # https://github.com/openshift/os/issues/1680
       # https://issues.redhat.com/browse/COS-3075
-      # NOTE: when reverting this, also revert the associated hack in get-ocp-repo.sh
       - rhel-9.4-appstream
     packages:
       - crun-wasm
diff --git a/extensions/Dockerfile b/extensions/Dockerfile
@@ -7,10 +7,14 @@ RUN mkdir /os
 WORKDIR /os
 ADD . .
 ARG OPENSHIFT_CI=0
-RUN if [ "${OPENSHIFT_CI}" != 0 ]; then ci/get-ocp-repo.sh --ocp-layer packages-openshift.yaml; fi
 RUN --mount=type=secret,id=yumrepos,target=/os/secret.repo <<EOF
 set -xeuo pipefail
 
+# fetch repos from in-cluster mirrors if we're running in OpenShift CI
+if [ "${OPENSHIFT_CI}" != 0 ]; then
+  /run/src/ci/get-ocp-repo.sh ocp.repo
+fi
+
 . /etc/os-release
 if [ $ID = rhel ]; then
   MANIFEST="manifest-rhel-9.6.yaml"

Original file line number	Diff line number	Diff line change
`@@ -60,7 +60,7 @@ cosa_init() {`
`60`	`60`
`61`	`61`	`# Initialize the .repo files`
`62`	`62`	`prepare_repos() {`
`63`		`- src/config/ci/get-ocp-repo.sh --cosa-workdir .`
	`63`	`+ src/config/ci/get-ocp-repo.sh src/config/ocp.repo`
`64`	`64`	`}`
`65`	`65`
`66`	`66`	`# Do a cosa build & cosa build-extensions only.`