ROX-25508: migrate collector pipeline to matrix builds (#2634) (#2644)

Author: tommartensen

.tekton/collector-build.yaml

@@ -58,19 +58,8 @@ spec:
       secretName: '{{ git_auth_secret }}'
 
   taskRunSpecs:
-
-  # Only adjusting computeResources for amd64 build because
-  # multi-arch builds happen off cluster
-  - pipelineTaskName: build-container-amd64
+  - pipelineTaskName: build-images
     stepSpecs:
-    - name: build
-      # CPU requests are increased to speed up builds compared to the defaults.
-      # Defaults: https://github.com/redhat-appstudio/build-definitions/blob/main/task/buildah/0.1/buildah.yaml#L126
-      computeResources:
-        limits:
-          cpu: 4
-        requests:
-          cpu: 4
     - name: use-trusted-artifact
       # use-/create-trusted-artifact gets OOM-killed when a cluster is loaded. Bigger mem limits==request should help.
       computeResources: &ta-resources
@@ -105,18 +94,6 @@ spec:
         requests:
           cpu: "1"
           memory: 6Gi
-  - pipelineTaskName: build-container-s390x
-    stepSpecs:
-    - name: use-trusted-artifact
-      computeResources: *ta-resources
-  - pipelineTaskName: build-container-ppc64le
-    stepSpecs:
-    - name: use-trusted-artifact
-      computeResources: *ta-resources
-  - pipelineTaskName: build-container-arm64
-    stepSpecs:
-    - name: use-trusted-artifact
-      computeResources: *ta-resources
   - pipelineTaskName: build-source-image
     stepSpecs:
     - name: use-trusted-artifact

.tekton/collector-component-pipeline.yaml

@@ -114,6 +114,17 @@ spec:
   - name: extra-labels
     type: array
     description: Additional labels to put on the built containers.
+  - default:
+    - linux-c2xlarge/amd64
+    - linux-c2xlarge/arm64
+    - linux/ppc64le
+    - linux/s390x
+    description: >
+      List of platforms to build the container images for. The available
+      set of values is determined by the configuration of the multi-platform-controller
+      on the cluster: https://konflux.pages.redhat.com/docs/users/getting-started/multi-platform-builds.html
+    name: build-platforms
+    type: array
   results:
   - description: ""
     name: IMAGE_URL
@@ -239,148 +250,17 @@ spec:
     workspaces:
     - name: git-basic-auth
       workspace: git-auth
-  - name: build-container-amd64
-    params:
-    - name: IMAGE
-      value: $(params.output-image-repo):$(tasks.determine-image-tag.results.IMAGE_TAG)-amd64
-    - name: DOCKERFILE
-      value: $(params.dockerfile)
-    - name: CONTEXT
-      value: $(params.path-context)
-    - name: HERMETIC
-      value: $(params.hermetic)
-    - name: PREFETCH_INPUT
-      value: $(params.prefetch-input)
-    - name: IMAGE_EXPIRES_AFTER
-      value: $(tasks.determine-image-expiration.results.IMAGE_EXPIRES_AFTER)
-    - name: COMMIT_SHA
-      value: $(tasks.clone-repository.results.commit)
-    - name: TARGET_STAGE
-      value: $(params.build-target-stage)
-    - name: BUILD_ARGS
-      value:
-      - COLLECTOR_TAG=$(tasks.determine-image-tag.results.IMAGE_TAG)
-    - name: SOURCE_ARTIFACT
-      value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT)
-    - name: CACHI2_ARTIFACT
-      value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT)
-    - name: BUILDAH_FORMAT
-      value: $(params.buildah-format)
-    - name: LABELS
-      value: ["$(params.extra-labels[*])"]
-    - name: BUILD_TIMESTAMP
-      value: "$(tasks.clone-repository.results.commit-timestamp)"
-    taskRef:
+  - name: build-images
+    matrix:
       params:
-      - name: name
-        value: buildah-oci-ta
-      - name: bundle
-        value: quay.io/konflux-ci/tekton-catalog/task-buildah-oci-ta:0.6@sha256:80c6d4217aab2344511e4cf18ae7570878e411b1c40e5369a26263cf4adcef30
-      - name: kind
-        value: task
-      resolver: bundles
-    when:
-    - input: $(tasks.init.results.build)
-      operator: in
-      values: ["true"]
-  - name: build-container-s390x
+      - name: PLATFORM
+        value:
+        - $(params.build-platforms)
     params:
     - name: IMAGE
-      value: $(params.output-image-repo):$(tasks.determine-image-tag.results.IMAGE_TAG)-s390x
-    - name: DOCKERFILE
-      value: $(params.dockerfile)
-    - name: CONTEXT
-      value: $(params.path-context)
-    - name: HERMETIC
-      value: $(params.hermetic)
-    - name: PREFETCH_INPUT
-      value: $(params.prefetch-input)
-    - name: IMAGE_EXPIRES_AFTER
-      value: $(tasks.determine-image-expiration.results.IMAGE_EXPIRES_AFTER)
-    - name: COMMIT_SHA
-      value: $(tasks.clone-repository.results.commit)
-    - name: TARGET_STAGE
-      value: $(params.build-target-stage)
-    - name: BUILD_ARGS
-      value:
-      - COLLECTOR_TAG=$(tasks.determine-image-tag.results.IMAGE_TAG)
-    - name: SOURCE_ARTIFACT
-      value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT)
-    - name: CACHI2_ARTIFACT
-      value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT)
-    - name: PLATFORM
-      value: linux/s390x
-    - name: BUILDAH_FORMAT
-      value: $(params.buildah-format)
-    - name: LABELS
-      value: ["$(params.extra-labels[*])"]
-    - name: BUILD_TIMESTAMP
-      value: "$(tasks.clone-repository.results.commit-timestamp)"
-    taskRef:
-      params:
-      - name: name
-        value: buildah-remote-oci-ta
-      - name: bundle
-        value: quay.io/konflux-ci/tekton-catalog/task-buildah-remote-oci-ta:0.6@sha256:09a9f4bffb30192487e03e6563ca65a209a41b92f18bd9f491763794d7cc0efe
-      - name: kind
-        value: task
-      resolver: bundles
-    when:
-    - input: $(tasks.init.results.build)
-      operator: in
-      values: ["true"]
-    timeout: 1h30m0s
-  - name: build-container-ppc64le
-    params:
-    - name: IMAGE
-      value: $(params.output-image-repo):$(tasks.determine-image-tag.results.IMAGE_TAG)-ppc64le
-    - name: DOCKERFILE
-      value: $(params.dockerfile)
-    - name: CONTEXT
-      value: $(params.path-context)
-    - name: HERMETIC
-      value: $(params.hermetic)
-    - name: PREFETCH_INPUT
-      value: $(params.prefetch-input)
-    - name: IMAGE_EXPIRES_AFTER
-      value: $(tasks.determine-image-expiration.results.IMAGE_EXPIRES_AFTER)
-    - name: COMMIT_SHA
-      value: $(tasks.clone-repository.results.commit)
-    - name: TARGET_STAGE
-      value: $(params.build-target-stage)
-    - name: BUILD_ARGS
-      value:
-      - COLLECTOR_TAG=$(tasks.determine-image-tag.results.IMAGE_TAG)
-    - name: SOURCE_ARTIFACT
-      value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT)
-    - name: CACHI2_ARTIFACT
-      value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT)
-    - name: PLATFORM
-      value: linux/ppc64le
-    - name: BUILDAH_FORMAT
-      value: $(params.buildah-format)
-    - name: LABELS
-      value: ["$(params.extra-labels[*])"]
-    - name: BUILD_TIMESTAMP
-      value: "$(tasks.clone-repository.results.commit-timestamp)"
-    taskRef:
-      params:
-      - name: name
-        value: buildah-remote-oci-ta
-      - name: bundle
-        value: quay.io/konflux-ci/tekton-catalog/task-buildah-remote-oci-ta:0.6@sha256:09a9f4bffb30192487e03e6563ca65a209a41b92f18bd9f491763794d7cc0efe
-      - name: kind
-        value: task
-      resolver: bundles
-    when:
-    - input: $(tasks.init.results.build)
-      operator: in
-      values: ["true"]
-    timeout: 1h30m0s
-  - name: build-container-arm64
-    params:
-    - name: IMAGE
-      value: $(params.output-image-repo):$(tasks.determine-image-tag.results.IMAGE_TAG)-arm64
+      value: $(params.output-image-repo):$(tasks.determine-image-tag.results.IMAGE_TAG)
+    - name: IMAGE_APPEND_PLATFORM
+      value: "true"
     - name: DOCKERFILE
       value: $(params.dockerfile)
     - name: CONTEXT
@@ -402,8 +282,6 @@ spec:
       value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT)
     - name: CACHI2_ARTIFACT
       value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT)
-    - name: PLATFORM
-      value: linux-c2xlarge/arm64
     - name: BUILDAH_FORMAT
       value: $(params.buildah-format)
     - name: LABELS
@@ -432,14 +310,12 @@ spec:
       value: $(tasks.clone-repository.results.commit)
     - name: IMAGES
       value:
-      - $(tasks.build-container-amd64.results.IMAGE_REF)
-      - $(tasks.build-container-s390x.results.IMAGE_REF)
-      - $(tasks.build-container-ppc64le.results.IMAGE_REF)
-      - $(tasks.build-container-arm64.results.IMAGE_REF)
+      - $(tasks.build-images.results.IMAGE_REF[*])
     - name: IMAGE_EXPIRES_AFTER
       value: $(tasks.determine-image-expiration.results.IMAGE_EXPIRES_AFTER)
     - name: BUILDAH_FORMAT
       value: $(params.buildah-format)
+    runAfter: [ build-images ]
     taskRef:
       params:
       - name: name
@@ -521,6 +397,11 @@ spec:
       operator: in
       values: ["false"]
   - name: clair-scan
+    matrix:
+      params:
+      - name: image-platform
+        value:
+        - $(params.build-platforms)
     params:
     - name: image-digest
       value: $(tasks.build-image-index.results.IMAGE_DIGEST)
@@ -540,6 +421,11 @@ spec:
       operator: in
       values: ["false"]
   - name: ecosystem-cert-preflight-checks
+    matrix:
+      params:
+      - name: platform
+        value:
+        - $(params.build-platforms)
     params:
     - name: image-url
       value: $(tasks.build-image-index.results.IMAGE_URL)
@@ -626,6 +512,11 @@ spec:
       operator: in
       values: ["false"]
   - name: clamav-scan
+    matrix:
+      params:
+      - name: image-arch
+        value:
+        - $(params.build-platforms)
     params:
     - name: image-digest
       value: $(tasks.build-image-index.results.IMAGE_DIGEST)