chore: migrate from kuttl to chainsaw

.chainsaw.yaml

@@ -0,0 +1,18 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/configuration-chainsaw-v1alpha1.json
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Configuration
+metadata:
+  creationTimestamp: null
+  name: configuration
+spec:
+  parallel: 1
+  reportFormat: JSON
+  reportName: e2e
+  failFast: true
+  timeouts:
+    apply: 5m0s
+    assert: 5m0s
+    cleanup: 5m0s
+    delete: 5m0s
+    error: 5m0s
+    exec: 5m0s

.github/workflows/ci-build.yaml

@@ -49,19 +49,8 @@ jobs:
         k3s-version: [ v1.27.1 ]
         # k3s-version: [v1.20.2, v1.19.2, v1.18.9, v1.17.11, v1.16.15]
     steps:
-      - name: Download kuttl plugin
-        env:
-          KUTTL_VERSION: '0.10.0'
-          KUTTL_PLUGIN_SUM: 'ad21c0d2be495a8f6cfc0821e592ae61afd191ebd453156d9b204e2524fceaf2'
-          KUTTL_PLUGIN_FILENAME: 'kubectl-kuttl_0.10.0_linux_x86_64'
-        run: |
-          set -x
-          echo ${KUTTL_PLUGIN_FILENAME}
-          curl -OL https://github.com/kudobuilder/kuttl/releases/download/v${KUTTL_VERSION}/${KUTTL_PLUGIN_FILENAME}
-          echo "${KUTTL_PLUGIN_SUM}  ${KUTTL_PLUGIN_FILENAME}" | sha256sum -c -
-          sudo mv ${KUTTL_PLUGIN_FILENAME} /usr/local/bin/kubectl-kuttl
-          sudo chmod +x /usr/local/bin/kubectl-kuttl
-          kubectl-kuttl version
+      - name: Install chainsaw
+        uses: kyverno/[email protected]
       - name: Install K3D
         run: |
           set -x

.gitignore

@@ -29,8 +29,6 @@ vendor/
 # transient build artifacts
 build/_output
 
-kubeconfig
-kuttl-test.json
 e2e.json
 
 # Desktop Services Store in macOS system

Makefile

@@ -125,7 +125,7 @@ undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/confi
 ##@ E2E
 
 e2e: ## Run operator e2e tests
-	kubectl kuttl test ./tests/k8s --config ./tests/kuttl-tests.yaml 
+	chainsaw test ./tests/k8s
 
 all: test install run e2e ## UnitTest, Run the operator locally and execute e2e tests.
 

README.md

@@ -13,7 +13,7 @@ See the [documentation][docs] for installation and usage of the operator.
 
 ## E2E testing
 
-E2E tests are written using [KUTTL](https://kuttl.dev/docs/#install-kuttl-cli). Please Install [KUTTL](https://kuttl.dev/docs/#install-kuttl-cli) to run the tests.
+E2E tests are written using [chainsaw](https://kyverno.github.io/chainsaw). Please Install [chainsaw](https://kyverno.github.io/chainsaw/latest/install/) to run the tests.
 
 Note that the e2e tests for Redis HA mode require a cluster with at least three worker nodes.  A local three-worker node
 cluster can be created using [k3d](https://k3d.io/)

docs/e2e-test-guide.md

@@ -1,14 +1,14 @@
 # Argo CD Operator E2E Test Guide
 
-E2E tests are written using [KUTTL](https://kuttl.dev/docs/#install-kuttl-cli).
+E2E tests are written using [chainsaw](https://kyverno.github.io/chainsaw/latest/install/).
 
 ## Requirements
 
 This test suite assumes that an Argo CD Operator is installed on the cluster or running locally using `ARGOCD_CLUSTER_CONFIG_NAMESPACES=argocd-e2e-cluster-config make install run`.
 
 The system executing the tests must have following tools installed:
 
-* `kuttl` kubectl plugin (>= v0.11.1)
+* `chainsaw`
 * `oc` and `kubectl` client
 * `jq` for parsing JSON data
 * `curl`
@@ -47,28 +47,28 @@ make e2e
 make all
 ```
 
-### Running manual with kuttl
+### Running manual with chainsaw
 
 ```sh
-kubectl kuttl test ./tests/k8s --config ./tests/kuttl-tests.yaml
+chainsaw test ./tests/k8s
 ```
 
 ### Running single tests
 
 Sometimes (e.g. when initially writing a test or troubleshooting an existing
 one), you may want to run single test cases isolated. To do so, you can pass
-the name of the test using `--test` to `kuttl`, i.e.
+the test folder to chainsaw, i.e.
 
 ```sh
-kubectl kuttl test ./tests/k8s --config ./tests/kuttl-tests.yaml --test 1-004_validate_namespace_scoped_install
+chainsaw test ./tests/k8s/1-004_validate_namespace_scoped_install
 ```
 
 The name of the test is the name of the directory containing its steps and
 assertions.
 
-If you are troubleshooting, you may want to prevent `kuttl` from deleting the
-test's namespace afterwards. In order to do so, just pass the additional flag
-`--skip-delete` to above command.
+If you are troubleshooting, you may want to prevent `chainsaw` from deleting the
+test's namespace afterward. To do so, just pass the additional flag
+`--skip-delete` to the above command.
 
 ## Writing new tests
 
@@ -84,22 +84,8 @@ The `<test ID>` is the serial number of the test as defined in the Test Plan
 document. The `<short description>` is exactly that, a short description of
 what happens in the test.
 
-### Name of the test steps
-
-Each test step is a unique YAML file within the test's directory. The name of
-the step is defined by its file name.
-
-The test steps must be named `XX-<name>.yaml`. This is a `kuttl` convention
-and cannot be overriden. `XX` is a number (prefixed with `0`, so step `1` must
-be `01`), and `<name>` is a free form value for the test step.
-
-There are two reserved words you cannot use for `<name>`:
-
-* `assert` contains positive assertions (i.e. resources that must exist) and
-* `errors` contains negative assertions (i.e. resources that must not exist)
-
 Refer to the
-[kuttl documentation](https://kuttl.dev/docs)
+[chainsaw documentation](https://kyverno.github.io/chainsaw/latest/writing-tests/)
 for more information.
 
 ### Documentation
@@ -112,45 +98,49 @@ will help others in troubleshooting failing tests.
 
 ### Recipes
 
-`kuttl` unfortunately neither encourages or supports re-use of your test steps
-and assertions yet.
-
 Generally, you should try to use `assert` and `errors` declaration whenever
 possible and viable. For some cases, you may need to use custom scripts to
 get the results you are looking for.
 
 #### Scripts general
 
-Scripts can be executed in a `kuttl.dev/TestStep` resources from a usual test
-step declaration.
+Scripts can be executed in a `Test` resource.
 
 Your script probably will retrieve some information, and asserts it state. If
 the assertion fails, the script should exit with a code > 0, and also print
 some information why it failed, e.g.
 
 ```yaml
-apiVersion: kuttl.dev/v1beta1
-kind: TestStep
-commands:
-- script: |
-    # Get some piece of information...
-    if test "$result" != "expected"; then
-      echo "Expectation failed, should 'expected', is '$result'"
-      exit 1
-    fi
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  name: example
+spec:
+  steps:
+  - script:
+      content: |
+        # Get some piece of information...
+        if test "$result" != "expected"; then
+          echo "Expectation failed, should 'expected', is '$result'"
+          exit 1
+        fi
 ```
 
 Also, you may want to use `set -e` and `set -o pipefail` at the top of your
 script to catch unexpected errors as test case failures, e.g.
 
 ```yaml
-apiVersion: kuttl.dev/v1beta1
-kind: TestStep
-commands:
-- script: |
-    set -e
-    set -o pipefail
-    # rest of your script
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  name: example
+spec:
+  steps:
+  - script:
+      content: |
+        set -e
+        set -o pipefail
+        # rest of your script
 ```
 
 #### Getting values of a resource's environment variables
@@ -165,14 +155,18 @@ of a variable named `FOO` for the `argocd-server` deployment in the test's
 namespace:
 
 ```yaml
-apiVersion: kuttl.dev/v1beta1
-kind: TestStep
-commands:
-- script: |
-    val=$(kubectl get -n $NAMESPACE deployments argocd-server -o json \
-      | jq -r '.spec.templates.spec.containers[0].env[]|select(.name=="FOO").value')
-    if test "$val" != "bar"; then
-      echo "Expectation failed for for env FOO in argocd-server: should 'bar', is '$val'"
-      exit 1
-    fi
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  name: example
+spec:
+  steps:
+  - script:
+      content: |
+        val=$(kubectl get -n $NAMESPACE deployments argocd-server -o json \
+          | jq -r '.spec.templates.spec.containers[0].env[]|select(.name=="FOO").value')
+        if test "$val" != "bar"; then
+          echo "Expectation failed for for env FOO in argocd-server: should 'bar', is '$val'"
+          exit 1
+        fi
 ```

hack/test.sh

@@ -18,4 +18,4 @@
 
 set -e 
 
-kubectl kuttl test ./tests/k8s ./tests/ha --config ./tests/kuttl-tests.yaml 
+chainsaw test ./tests/k8s ./tests/ha

tests/ha/1-020_validate_redis_ha_nonha/01-assert.yaml

@@ -1,7 +1,3 @@
-apiVersion: kuttl.dev/v1beta1
-kind: TestAssert
-timeout: 720
----
 apiVersion: v1
 kind: Service
 metadata:
@@ -20,4 +16,4 @@ kind: Deployment
 metadata:
   name: example-argocd-redis
 status:
-  readyReplicas: 1
+  readyReplicas: 1

tests/ha/1-020_validate_redis_ha_nonha/02-assert.yaml

@@ -1,7 +1,3 @@
-apiVersion: kuttl.dev/v1beta1
-kind: TestAssert
-timeout: 720
----
 apiVersion: v1
 kind: Service
 metadata:
@@ -45,9 +41,9 @@ spec:
       affinity:
         podAntiAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
-            - labelSelector:
-                matchLabels:
-                  app.kubernetes.io/name: example-argocd-redis-ha
+          - labelSelector:
+              matchLabels:
+                app.kubernetes.io/name: example-argocd-redis-ha
 status:
   readyReplicas: 3
 ---
@@ -56,4 +52,4 @@ kind: Deployment
 metadata:
   name: example-argocd-redis-ha-haproxy
 status:
-  readyReplicas: 1
+  readyReplicas: 1

tests/ha/1-020_validate_redis_ha_nonha/03-assert.yaml

@@ -1,7 +1,3 @@
-apiVersion: kuttl.dev/v1beta1
-kind: TestAssert
-timeout: 720
----
 apiVersion: argoproj.io/v1alpha1
 kind: ArgoCD
 metadata:
@@ -73,5 +69,3 @@ spec:
             memory: 128Mi
 status:
   readyReplicas: 3
-
-

tests/ha/1-020_validate_redis_ha_nonha/04-assert.yaml

@@ -1,7 +1,3 @@
-apiVersion: kuttl.dev/v1beta1
-kind: TestAssert
-timeout: 720
----
 apiVersion: argoproj.io/v1alpha1
 kind: ArgoCD
 metadata:
@@ -15,4 +11,4 @@ kind: Deployment
 metadata:
   name: example-argocd-redis
 status:
-  readyReplicas: 1
+  readyReplicas: 1

tests/ha/1-020_validate_redis_ha_nonha/chainsaw-test.yaml

@@ -0,0 +1,38 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  creationTimestamp: null
+  name: 1-020-validate-redis-ha-nonha
+spec:
+  steps:
+  - name: step-01
+    try:
+    - apply:
+        file: 01-basic.yaml
+    - assert:
+        file: 01-assert.yaml
+    - error:
+        file: 01-errors.yaml
+  - name: step-02
+    try:
+    - apply:
+        file: 02-enable-ha.yaml
+    - assert:
+        file: 02-assert.yaml
+    - error:
+        file: 02-errors.yaml
+  - name: step-03
+    try:
+    - apply:
+        file: 03-update-ha-resources.yaml
+    - assert:
+        file: 03-assert.yaml
+  - name: step-04
+    try:
+    - apply:
+        file: 04-disable-ha.yaml
+    - assert:
+        file: 04-assert.yaml
+    - error:
+        file: 04-errors.yaml

tests/k8s/1-0017_validate_cmp/01-assert.yaml

@@ -1,9 +1,3 @@
-# Increase the timeout for the first test because it needs to download
-# a number of container images
-apiVersion: kuttl.dev/v1beta1
-kind: TestAssert
-timeout: 720
----
 apiVersion: argoproj.io/v1alpha1
 kind: ArgoCD
 metadata:
@@ -23,9 +17,8 @@ status:
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: guestbook
-  namespace: test-1-17-custom
   annotations:
     Bar: baz
     Foo: myfoo
-
+  name: guestbook
+  namespace: test-1-17-custom