chore: migrate from kuttl to chainsaw (#2630)

Author: eddycharly

.chainsaw.yaml

@@ -0,0 +1,12 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/configuration-chainsaw-v1alpha1.json
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Configuration
+metadata:
+  name: configuration
+spec:
+  parallel: 4
+  timeouts:
+    assert: 5m0s
+    cleanup: 5m0s
+    delete: 5m0s
+    error: 5m0s

.github/workflows/e2e.yaml

@@ -12,7 +12,6 @@ concurrency:
 
 jobs:
   e2e-tests:
-    name: End-to-end tests ${{ matrix.group }} on K8s ${{ matrix.kube-version }}
     runs-on: ubuntu-22.04
     strategy:
       fail-fast: false
@@ -25,13 +24,13 @@ jobs:
          - "1.29"
        group:
          - e2e
-         - e2e-instrumentation
-         - e2e-upgrade
          - e2e-autoscale
-         - e2e-pdb
+         - e2e-instrumentation
          - e2e-opampbridge
-         - e2e-targetallocator
+         - e2e-pdb
          - e2e-prometheuscr
+         - e2e-targetallocator
+         - e2e-upgrade
          - e2e-multi-instrumentation
        include:
          - group: e2e-prometheuscr
@@ -42,33 +41,38 @@ jobs:
     steps:
     - name: Check out code into the Go module directory
       uses: actions/checkout@v4
-
     - name: Set up Go
       uses: actions/setup-go@v5
       with:
         go-version: "~1.21.3"
-
     - name: Cache tools
       uses: actions/cache@v4
       with:
         path: bin
         key: ${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('Makefile') }}
-
+    - name: Install chainsaw
+      uses: kyverno/[email protected]
     - name: Install tools
       run: make install-tools
-
-    - name: "run tests"
+    - name: Prepare e2e tests
       env:
         KUBE_VERSION: ${{ matrix.kube-version }}
       run: |
+        set -e
         make ${{ matrix.setup != '' && matrix.setup || 'prepare-e2e' }} KUBE_VERSION=$KUBE_VERSION VERSION=e2e
+    - name: Run e2e tests
+      env:
+        KUBE_VERSION: ${{ matrix.kube-version }}
+      run: |
+        set -e
         make ${{ matrix.group }}
-
     - name: "log operator if failed"
       if: ${{ failure() }}
       env:
         KUBE_VERSION: ${{ matrix.kube-version }}
-      run: make e2e-log-operator KUBE_VERSION=$KUBE_VERSION
+      run: |
+        set -e
+        make e2e-log-operator KUBE_VERSION=$KUBE_VERSION
 
   e2e-tests-check:
     runs-on: ubuntu-22.04

CONTRIBUTING.md

@@ -118,11 +118,11 @@ KUBEBUILDER_ASSETS=$(./bin/setup-envtest use -p path 1.23) go test ./pkg...
 
 ### End to end tests
 
-To run the end-to-end tests, you'll need [`kind`](https://kind.sigs.k8s.io) and [`kuttl`](https://kuttl.dev). Refer to their documentation for installation instructions.
+To run the end-to-end tests, you'll need [`kind`](https://kind.sigs.k8s.io) and [`chainsaw`](https://kyverno.github.io/chainsaw). Refer to their documentation for installation instructions.
 
 Once they are installed, the tests can be executed with `make prepare-e2e`, which will build an image to use with the tests, followed by `make e2e`. Each call to the `e2e` target will setup a fresh `kind` cluster, making it safe to be executed multiple times with a single `prepare-e2e` step.
 
-The tests are located under `tests/e2e` and are written to be used with `kuttl`. Refer to their documentation to understand how tests are written.
+The tests are located under `tests/e2e` and are written to be used with `chainsaw`. Refer to their documentation to understand how tests are written.
 
 To evert the changes made by the `make prepare-e2e` run `make reset`.
 
@@ -133,7 +133,7 @@ To install the OpenTelemetry operator, please follow the instructions in  [Opera
 
 Once the operator is installed, the tests can be executed using `make e2e-openshift`, which will call to the `e2e-openshift` target. Note that `kind` is disabled for the TestSuite as the requirement is to use an OpenShift cluster for these test cases. 
 
-The tests are located under `tests/e2e-openshift` and are written to be used with `kuttl`.
+The tests are located under `tests/e2e-openshift` and are written to be used with `chainsaw`.
 
 ### Undeploying the operator from the local cluster
 

Makefile

@@ -197,39 +197,18 @@ generate: controller-gen
 
 # end-to-tests
 .PHONY: e2e
-e2e:
-	$(KUTTL) test
-
-
-# instrumentation end-to-tests
-.PHONY: e2e-instrumentation
-e2e-instrumentation:
-	$(KUTTL) test --config kuttl-test-instrumentation.yaml
-
-# end-to-end-test for PrometheusCR E2E tests
-.PHONY: e2e-prometheuscr
-e2e-prometheuscr:
-	$(KUTTL) test --config kuttl-test-prometheuscr.yaml
-
-# end-to-end-test for testing upgrading
-.PHONY: e2e-upgrade
-e2e-upgrade: undeploy
-	$(KUTTL) test --config kuttl-test-upgrade.yaml
+e2e: chainsaw
+	$(CHAINSAW) test --test-dir ./tests/e2e
 
 # end-to-end-test for testing autoscale
 .PHONY: e2e-autoscale
-e2e-autoscale:
-	$(KUTTL) test --config kuttl-test-autoscale.yaml
-
-# end-to-end-test for testing pdb support
-.PHONY: e2e-pdb
-e2e-pdb:
-	$(KUTTL) test --config kuttl-test-pdb.yaml
+e2e-autoscale: chainsaw
+	$(CHAINSAW) test --test-dir ./tests/e2e-autoscale
 
-# end-to-end-test for testing OpenShift cases
-.PHONY: e2e-openshift
-e2e-openshift:
-	$(KUTTL) test --config kuttl-test-openshift.yaml
+# instrumentation end-to-tests
+.PHONY: e2e-instrumentation
+e2e-instrumentation: chainsaw
+	$(CHAINSAW) test --test-dir ./tests/e2e-instrumentation
 
 .PHONY: e2e-log-operator
 e2e-log-operator:
@@ -238,24 +217,41 @@ e2e-log-operator:
 
 # end-to-tests for multi-instrumentation
 .PHONY: e2e-multi-instrumentation
-e2e-multi-instrumentation:
-	$(KUTTL) test --config kuttl-test-multi-instr.yaml
+e2e-multi-instrumentation: chainsaw
+	$(CHAINSAW) test --test-dir ./tests/e2e-multi-instrumentation
 
 # OpAMPBridge CR end-to-tests
 .PHONY: e2e-opampbridge
-e2e-opampbridge:
-	$(KUTTL) test --config kuttl-test-opampbridge.yaml
+e2e-opampbridge: chainsaw
+	$(CHAINSAW) test --test-dir ./tests/e2e-opampbridge
+
+# end-to-end-test for testing pdb support
+.PHONY: e2e-pdb
+e2e-pdb: chainsaw
+	$(CHAINSAW) test --test-dir ./tests/e2e-pdb
+
+# end-to-end-test for PrometheusCR E2E tests
+.PHONY: e2e-prometheuscr
+e2e-prometheuscr: chainsaw
+	$(CHAINSAW) test --test-dir ./tests/e2e-prometheuscr
 
 # Target allocator end-to-tests
 .PHONY: e2e-targetallocator
-e2e-targetallocator:
-	$(KUTTL) test --config kuttl-test-targetallocator.yaml
+e2e-targetallocator: chainsaw
+	$(CHAINSAW) test --test-dir ./tests/e2e-targetallocator
+
+# end-to-end-test for testing upgrading
+.PHONY: e2e-upgrade
+e2e-upgrade: undeploy chainsaw
+	kubectl apply -f ./tests/e2e-upgrade/upgrade-test/opentelemetry-operator-v0.86.0.yaml
+	go run hack/check-operator-ready.go
+	$(CHAINSAW) test --test-dir ./tests/e2e-upgrade
 
 .PHONY: prepare-e2e
-prepare-e2e: kuttl set-image-controller add-image-targetallocator add-image-opampbridge container container-target-allocator container-operator-opamp-bridge start-kind cert-manager install-metrics-server install-targetallocator-prometheus-crds load-image-all deploy
+prepare-e2e: chainsaw set-image-controller add-image-targetallocator add-image-opampbridge container container-target-allocator container-operator-opamp-bridge start-kind cert-manager install-metrics-server install-targetallocator-prometheus-crds load-image-all deploy
 
 .PHONY: prepare-e2e-with-featuregates
-prepare-e2e-with-featuregates: kuttl enable-operator-featuregates prepare-e2e
+prepare-e2e-with-featuregates: chainsaw enable-operator-featuregates prepare-e2e
 
 .PHONY: scorecard-tests
 scorecard-tests: operator-sdk
@@ -361,7 +357,6 @@ cmctl:
 
 KUSTOMIZE ?= $(LOCALBIN)/kustomize
 KIND ?= $(LOCALBIN)/kind
-KUTTL ?= $(LOCALBIN)/kubectl-kuttl
 CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen
 ENVTEST ?= $(LOCALBIN)/setup-envtest
 CHLOGGEN ?= $(LOCALBIN)/chloggen
@@ -371,10 +366,16 @@ KUSTOMIZE_VERSION ?= v5.0.3
 CONTROLLER_TOOLS_VERSION ?= v0.12.0
 GOLANGCI_LINT_VERSION ?= v1.54.0
 KIND_VERSION ?= v0.20.0
-KUTTL_VERSION ?= 0.15.0
+
+# Checks if chainsaw is in your PATH
+ifneq ($(shell which chainsaw),)
+CHAINSAW ?= $(shell which chainsaw)
+else
+CHAINSAW ?= $(LOCALBIN)/chainsaw
+endif
 
 .PHONY: install-tools
-install-tools: kustomize golangci-lint kind controller-gen envtest crdoc kuttl kind operator-sdk
+install-tools: kustomize golangci-lint kind controller-gen envtest crdoc kind operator-sdk chainsaw
 
 .PHONY: kustomize
 kustomize: ## Download kustomize locally if necessary.
@@ -403,6 +404,18 @@ CRDOC = $(shell pwd)/bin/crdoc
 crdoc: ## Download crdoc locally if necessary.
 	$(call go-get-tool,$(CRDOC), fybrik.io/crdoc,v0.5.2)
 
+.PHONY: chainsaw
+chainsaw: ## Find or download chainsaw
+ifeq (, $(shell which chainsaw))
+	@{ \
+	set -e ;\
+	go install github.com/kyverno/[email protected] ;\
+	}
+CHAINSAW ?= $(GOBIN)/chainsaw
+else
+CHAINSAW ?= $(shell which chainsaw)
+endif
+
 # go-get-tool will 'go get' any package $2 and install it to $1.
 PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST))))
 define go-get-tool
@@ -418,10 +431,6 @@ rm -rf $$TMP_DIR ;\
 }
 endef
 
-.PHONY: kuttl
-kuttl: $(LOCALBIN)
-	@KUTTL=$(KUTTL) KUTTL_VERSION=$(KUTTL_VERSION) ./hack/install-kuttl.sh
-
 OPERATOR_SDK = $(shell pwd)/bin/operator-sdk
 .PHONY: operator-sdk
 operator-sdk: $(LOCALBIN)