Use kustomize for e2e test setup (open-telemetry#2479)

swiatekm · web-flow · commit 0e60da4eabdd · 2024-01-08T15:15:36.000+01:00
* Use kustomize for setting target allocator image in E2E tests

* Use kustomize for setting opamp bridge image in E2E tests

* Use kustomize for setting feature gates in E2E tests

* Move operator flags to manager manifest
diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
@@ -12,7 +12,7 @@ concurrency:
 
 jobs:
   e2e-tests:
-    name: End-to-end tests
+    name: End-to-end tests ${{ matrix.group }} on K8s ${{ matrix.kube-version }}
     runs-on: ubuntu-22.04
     strategy:
       fail-fast: false
@@ -21,17 +21,22 @@ jobs:
        # All Kubernetes version in between expose the same APIs, hence the operator
        # should be compatible with them.
        kube-version:
-       - "1.23"
-       - "1.29"
+         - "1.23"
+         - "1.29"
        group:
          - e2e
          - e2e-instrumentation
          - e2e-upgrade
-         - e2e-prometheuscr
          - e2e-autoscale
-         - e2e-multi-instrumentation
          - e2e-pdb
          - e2e-opampbridge
+         - e2e-prometheuscr
+         - e2e-multi-instrumentation
+       include:
+         - group: e2e-prometheuscr
+           setup: "prepare-e2e-with-featuregates FEATUREGATES=+operator.observability.prometheus"
+         - group: e2e-multi-instrumentation
+           setup: "prepare-e2e-with-featuregates FEATUREGATES=+operator.autoinstrumentation.multi-instrumentation"
 
     steps:
     - name: Check out code into the Go module directory
@@ -57,7 +62,9 @@ jobs:
     - name: "run tests"
       env:
         KUBE_VERSION: ${{ matrix.kube-version }}
-      run: make prepare-e2e ${{ matrix.group }} KUBE_VERSION=$KUBE_VERSION VERSION=e2e
+      run: |
+        make ${{ matrix.setup != '' && matrix.setup || 'prepare-e2e' }} KUBE_VERSION=$KUBE_VERSION VERSION=e2e
+        make ${{ matrix.group }}
 
     - name: "log operator if failed"
       if: ${{ failure() }}
diff --git a/Makefile b/Makefile
@@ -1,10 +1,10 @@
 # Current Operator version
-VERSION ?= "$(shell git describe --tags | sed 's/^v//')"
+VERSION ?= $(shell git describe --tags | sed 's/^v//')
 VERSION_DATE ?= $(shell date -u +'%Y-%m-%dT%H:%M:%SZ')
 VERSION_PKG ?= "github.com/open-telemetry/opentelemetry-operator/internal/version"
 OTELCOL_VERSION ?= "$(shell grep -v '\#' versions.txt | grep opentelemetry-collector | awk -F= '{print $$2}')"
 OPERATOR_VERSION ?= "$(shell grep -v '\#' versions.txt | grep operator= | awk -F= '{print $$2}')"
-TARGETALLOCATOR_VERSION ?= "$(shell grep -v '\#' versions.txt | grep targetallocator | awk -F= '{print $$2}')"
+TARGETALLOCATOR_VERSION ?= $(shell grep -v '\#' versions.txt | grep targetallocator | awk -F= '{print $$2}')
 OPERATOR_OPAMP_BRIDGE_VERSION ?= "$(shell grep -v '\#' versions.txt | grep operator-opamp-bridge | awk -F= '{print $$2}')"
 AUTO_INSTRUMENTATION_JAVA_VERSION ?= "$(shell grep -v '\#' versions.txt | grep autoinstrumentation-java | awk -F= '{print $$2}')"
 AUTO_INSTRUMENTATION_NODEJS_VERSION ?= "$(shell grep -v '\#' versions.txt | grep autoinstrumentation-nodejs | awk -F= '{print $$2}')"
@@ -143,6 +143,23 @@ uninstall: manifests kustomize
 set-image-controller: manifests kustomize
 	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
 
+.PHONY: add-operator-arg
+add-operator-arg: PATCH = [{"op":"add","path":"/spec/template/spec/containers/0/args/-","value":"$(OPERATOR_ARG)"}]
+add-operator-arg: manifests kustomize
+	cd config/manager && $(KUSTOMIZE) edit add patch --kind Deployment --patch '$(PATCH)'
+
+.PHONY: add-image-targetallocator
+add-image-targetallocator:
+	@$(MAKE) add-operator-arg OPERATOR_ARG=--target-allocator-image=$(TARGETALLOCATOR_IMG)
+
+.PHONY: add-image-opampbridge
+add-image-opampbridge:
+	@$(MAKE) add-operator-arg OPERATOR_ARG=--operator-opamp-bridge-image=$(OPERATOROPAMPBRIDGE_IMG)
+
+.PHONY: enable-operator-featuregates
+enable-operator-featuregates: OPERATOR_ARG = --feature-gates=$(FEATUREGATES)
+enable-operator-featuregates: add-operator-arg
+
 # Deploy controller in the current Kubernetes context, configured in ~/.kube/config
 .PHONY: deploy
 deploy: set-image-controller
@@ -239,13 +256,10 @@ e2e-opampbridge:
 	$(KUTTL) test --config kuttl-test-opampbridge.yaml
 
 .PHONY: prepare-e2e
-prepare-e2e: kuttl set-image-controller container container-target-allocator container-operator-opamp-bridge start-kind cert-manager install-metrics-server install-targetallocator-prometheus-crds load-image-all deploy
-	TARGETALLOCATOR_IMG=$(TARGETALLOCATOR_IMG) OPERATOROPAMPBRIDGE_IMG=$(OPERATOROPAMPBRIDGE_IMG) OPERATOR_IMG=$(IMG) SED_BIN="$(SED)" ./hack/modify-test-images.sh
-
-.PHONY: enable-prometheus-feature-flag
-enable-prometheus-feature-flag:
-	$(SED) -i "s#--feature-gates=+operator.autoinstrumentation.go#--feature-gates=+operator.autoinstrumentation.go,+operator.observability.prometheus#g" config/default/manager_auth_proxy_patch.yaml
+prepare-e2e: kuttl set-image-controller add-image-targetallocator add-image-opampbridge container container-target-allocator container-operator-opamp-bridge start-kind cert-manager install-metrics-server install-targetallocator-prometheus-crds load-image-all deploy
 
+.PHONY: prepare-e2e-with-featuregates
+prepare-e2e-with-featuregates: kuttl enable-operator-featuregates prepare-e2e
 
 .PHONY: scorecard-tests
 scorecard-tests: operator-sdk
@@ -281,7 +295,7 @@ container-operator-opamp-bridge: operator-opamp-bridge
 .PHONY: start-kind
 start-kind:
 ifeq (true,$(START_KIND_CLUSTER))
-	kind create cluster --name $(KIND_CLUSTER_NAME) --config $(KIND_CONFIG)
+	kind create cluster --name $(KIND_CLUSTER_NAME) --config $(KIND_CONFIG) || true
 endif
 
 .PHONY: install-metrics-server
@@ -456,7 +470,6 @@ reset: kustomize operator-sdk manifests
 	$(OPERATOR_SDK) bundle validate ./bundle
 	./hack/ignore-createdAt-bundle.sh
 	git checkout config/manager/kustomization.yaml
-	OPERATOR_IMG=local/opentelemetry-operator:e2e TARGETALLOCATOR_IMG=local/opentelemetry-operator-targetallocator:e2e OPERATOROPAMPBRIDGE_IMG=local/opentelemetry-operator-opamp-bridge:e2e DEFAULT_OPERATOR_IMG=$(IMG) DEFAULT_TARGETALLOCATOR_IMG=$(TARGETALLOCATOR_IMG) DEFAULT_OPERATOROPAMPBRIDGE_IMG=$(OPERATOROPAMPBRIDGE_IMG) SED_BIN="$(SED)" ./hack/modify-test-images.sh
 
 # Build the bundle image, used only for local dev purposes
 .PHONY: bundle-build
diff --git a/config/default/manager_auth_proxy_patch.yaml b/config/default/manager_auth_proxy_patch.yaml
@@ -27,10 +27,3 @@ spec:
           requests:
             cpu: 5m
             memory: 64Mi
-      - name: manager
-        args:
-        - "--metrics-addr=127.0.0.1:8080"
-        - "--enable-leader-election"
-        - "--zap-log-level=info"
-        - "--zap-time-encoding=rfc3339nano"
-        - "--feature-gates=+operator.autoinstrumentation.go,+operator.autoinstrumentation.nginx"
diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml
@@ -28,7 +28,11 @@ spec:
     spec:
       containers:
       - args:
-        - --enable-leader-election
+        - "--metrics-addr=127.0.0.1:8080"
+        - "--enable-leader-election"
+        - "--zap-log-level=info"
+        - "--zap-time-encoding=rfc3339nano"
+        - "--feature-gates=+operator.autoinstrumentation.go,+operator.autoinstrumentation.nginx"
         image: controller
         name: manager
         livenessProbe:
diff --git a/hack/modify-test-images.sh b/hack/modify-test-images.sh
diff --git a/kuttl-test-multi-instr.yaml b/kuttl-test-multi-instr.yaml
@@ -1,9 +1,6 @@
 apiVersion: kuttl.dev/v1beta1
 kind: TestSuite
 artifactsDir: ./tests/_build/artifacts/
-commands:
-  - command: kubectl apply -f ./tests/e2e-multi-instrumentation/manager_deployment_feature_gate.yaml
-  - command: go run hack/check-operator-ready.go
 testDirs:
   - ./tests/e2e-multi-instrumentation/
 timeout: 150
diff --git a/kuttl-test-prometheuscr.yaml b/kuttl-test-prometheuscr.yaml
@@ -11,10 +11,6 @@
 # create a new one when the selector changed.
 apiVersion: kuttl.dev/v1beta1
 kind: TestSuite
-commands:
-  - command: make undeploy
-  - command: make enable-prometheus-feature-flag deploy install-prometheus-operator
-  - command: go run hack/check-operator-ready.go
 testDirs:
   - ./tests/e2e-prometheuscr/
 timeout: 300
diff --git a/tests/e2e-multi-instrumentation/manager_deployment_feature_gate.yaml b/tests/e2e-multi-instrumentation/manager_deployment_feature_gate.yaml
diff --git a/tests/e2e-opampbridge/opampbridge/00-install.yaml b/tests/e2e-opampbridge/opampbridge/00-install.yaml
@@ -26,7 +26,6 @@ kind: OpAMPBridge
 metadata:
   name: test
 spec:
-  image: "local/opentelemetry-operator-opamp-bridge:e2e"
   endpoint: ws://opamp-server:4320/v1/opamp
   capabilities:
     AcceptsOpAMPConnectionSettings: true
diff --git a/tests/e2e-prometheuscr/create-sm-prometheus-exporters/08-install.yaml b/tests/e2e-prometheuscr/create-sm-prometheus-exporters/08-install.yaml
@@ -72,7 +72,6 @@ spec:
   targetAllocator:
     serviceAccount: ta
     enabled: true
-    image: "local/opentelemetry-operator-targetallocator:e2e"
     prometheusCR:
       enabled: true
     observability:
diff --git a/tests/e2e/prometheus-config-validation/00-promreceiver-allocatorconfig.yaml b/tests/e2e/prometheus-config-validation/00-promreceiver-allocatorconfig.yaml
@@ -26,7 +26,6 @@ spec:
   mode: statefulset
   targetAllocator:
     enabled: true
-    image: "local/opentelemetry-operator-targetallocator:e2e"
     serviceAccount: ta
     prometheusCR:
       enabled: true
diff --git a/tests/e2e/prometheus-config-validation/01-promreceiver-labeldrop.yaml b/tests/e2e/prometheus-config-validation/01-promreceiver-labeldrop.yaml
@@ -9,7 +9,6 @@ spec:
   mode: statefulset
   targetAllocator:
     enabled: true
-    image: "local/opentelemetry-operator-targetallocator:e2e"
     serviceAccount: ta
     prometheusCR:
       enabled: true
diff --git a/tests/e2e/prometheus-config-validation/02-promreceiver-allocatorconfig-extra.yaml b/tests/e2e/prometheus-config-validation/02-promreceiver-allocatorconfig-extra.yaml
@@ -7,7 +7,6 @@ spec:
   mode: statefulset
   targetAllocator:
     enabled: true
-    image: "local/opentelemetry-operator-targetallocator:e2e"
     serviceAccount: ta
     prometheusCR:
       enabled: true
diff --git a/tests/e2e/prometheus-config-validation/03-promreceiver-nopromconfig.yaml b/tests/e2e/prometheus-config-validation/03-promreceiver-nopromconfig.yaml
@@ -7,7 +7,6 @@ spec:
   mode: statefulset
   targetAllocator:
     enabled: true
-    image: "local/opentelemetry-operator-targetallocator:e2e"
     serviceAccount: ta
     prometheusCR:
       enabled: true
diff --git a/tests/e2e/smoke-targetallocator/00-install.yaml b/tests/e2e/smoke-targetallocator/00-install.yaml
@@ -27,7 +27,6 @@ spec:
   targetAllocator:
     enabled: true
     serviceAccount: ta
-    image: "local/opentelemetry-operator-targetallocator:e2e"
     prometheusCR:
       enabled: true
   config: |
diff --git a/tests/e2e/smoke-targetallocator/01-change-ta-config.yaml b/tests/e2e/smoke-targetallocator/01-change-ta-config.yaml
@@ -7,7 +7,6 @@ spec:
   targetAllocator:
     enabled: true
     serviceAccount: ta
-    image: "local/opentelemetry-operator-targetallocator:e2e"
     prometheusCR:
       enabled: true
       serviceMonitorSelector:
diff --git a/tests/e2e/targetallocator-features/00-install.yaml b/tests/e2e/targetallocator-features/00-install.yaml
@@ -47,7 +47,6 @@ spec:
             storage: 1Gi
   targetAllocator:
     enabled: true
-    image: "local/opentelemetry-operator-targetallocator:e2e"
     serviceAccount: ta
     securityContext:
       runAsUser: 1000
diff --git a/tests/e2e/targetallocator-prometheuscr/00-install.yaml b/tests/e2e/targetallocator-prometheuscr/00-install.yaml
@@ -104,7 +104,6 @@ spec:
   targetAllocator:
     enabled: true
     serviceAccount: ta
-    image: "local/opentelemetry-operator-targetallocator:e2e"
     prometheusCR:
       enabled: true
   config: |
