Fix helm chart issues (#17)

* fixes on the helm chart
added helm chart integration test

Signed-off-by: raffaelespazzoli <[email protected]>

* helm test successful

Signed-off-by: raffaelespazzoli <[email protected]>

* tested olm deployment

Signed-off-by: raffaelespazzoli <[email protected]>

Author: raffaelespazzoli

.github/workflows/pr.yaml

@@ -8,4 +8,8 @@ on:
 jobs:
   shared-operator-workflow:
     name: shared-operator-workflow
-    uses: redhat-cop/github-workflows-operators/.github/workflows/[email protected]
+    uses: redhat-cop/github-workflows-operators/.github/workflows/[email protected]
+    with: 
+      RUN_UNIT_TESTS: true
+      RUN_INTEGRATION_TESTS: false
+      RUN_HELMCHART_TEST: true

.github/workflows/push.yaml

@@ -10,10 +10,13 @@ on:
 jobs:
   shared-operator-workflow:
     name: shared-operator-workflow
-    uses: redhat-cop/github-workflows-operators/.github/workflows/[email protected].0
+    uses: redhat-cop/github-workflows-operators/.github/workflows/[email protected].3
     secrets:
       COMMUNITY_OPERATOR_PAT: ${{ secrets.COMMUNITY_OPERATOR_PAT }}
       REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }}
       REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
     with:
       PR_ACTOR: "[email protected]"
+      RUN_UNIT_TESTS: true
+      RUN_INTEGRATION_TESTS: false
+      RUN_HELMCHART_TEST: true

Makefile

@@ -1,6 +1,10 @@
 CHART_REPO_URL ?= http://example.com
 HELM_REPO_DEST ?= /tmp/gh-pages
 OPERATOR_NAME ?=$(shell basename -z `pwd`)
+HELM_VERSION ?= v3.8.0
+KIND_VERSION ?= v0.11.1
+KUBECTL_VERSION ?= v1.21.1
+VAULT_VERSION ?= 1.9.3
 
 # VERSION defines the project version for the bundle.
 # Update this value when you upgrade the version of your project.
@@ -94,6 +98,13 @@ vet: ## Run go vet against code.
 test: manifests generate fmt vet envtest ## Run tests.
 	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" go test ./... -coverprofile cover.out
 
+.PHONY: kind-setup
+kind-setup: kind kubectl helm
+	$(KIND) delete cluster
+	$(KIND) create cluster --image docker.io/kindest/node:$(KUBECTL_VERSION) --config=./integration/cluster-kind.yaml
+	$(HELM) upgrade ingress-nginx ./integration/helm/ingress-nginx -i --create-namespace -n ingress-nginx --atomic
+	$(KUBECTL) wait --namespace ingress-nginx --for=condition=ready pod --selector=app.kubernetes.io/component=controller --timeout=90s
+
 ##@ Build
 
 build: generate fmt vet ## Build manager binary.
@@ -110,18 +121,18 @@ docker-push: ## Push docker image with the manager.
 
 ##@ Deployment
 
-install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config.
-	$(KUSTOMIZE) build config/crd | kubectl apply -f -
+install: manifests kustomize kubectl ## Install CRDs into the K8s cluster specified in ~/.kube/config.
+	$(KUSTOMIZE) build config/crd | $(KUBECTL) apply -f -
 
-uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config.
-	$(KUSTOMIZE) build config/crd | kubectl delete -f -
+uninstall: manifests kustomize kubectl ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config.
+	$(KUSTOMIZE) build config/crd | $(KUBECTL) delete -f -
 
-deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config.
+deploy: manifests kustomize kubectl ## Deploy controller to the K8s cluster specified in ~/.kube/config.
 	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
-	$(KUSTOMIZE) build config/default | kubectl apply -f -
+	$(KUSTOMIZE) build config/default | $(KUBECTL) apply -f -
 
-undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config.
-	$(KUSTOMIZE) build config/default | kubectl delete -f -
+undeploy: kustomize kubectl ## Undeploy controller from the K8s cluster specified in ~/.kube/config.
+	$(KUSTOMIZE) build config/default | $(KUBECTL) delete -f -
 
 
 CONTROLLER_GEN = $(shell pwd)/bin/controller-gen
@@ -207,7 +218,7 @@ catalog-push: ## Push a catalog image.
 	$(MAKE) docker-push IMG=$(CATALOG_IMG)
 
 # Generate helm chart
-helmchart: kustomize
+helmchart: kustomize helm
 	mkdir -p ./charts/${OPERATOR_NAME}/templates
 	mkdir -p ./charts/${OPERATOR_NAME}/crds
 	repo=${OPERATOR_NAME} envsubst < ./config/local-development/tilt/env-replace-image.yaml > ./config/local-development/tilt/replace-image.yaml
@@ -220,15 +231,80 @@ helmchart: kustomize
 	version=${VERSION} image_repo=$${IMG%:*} envsubst < ./config/helmchart/values.yaml.tpl  > ./charts/${OPERATOR_NAME}/values.yaml
 	sed -i '1s/^/{{ if .Values.enableMonitoring }}/' ./charts/${OPERATOR_NAME}/templates/monitoring.coreos.com_v1_servicemonitor_${OPERATOR_NAME}-controller-manager-metrics-monitor.yaml
 	echo {{ end }} >> ./charts/${OPERATOR_NAME}/templates/monitoring.coreos.com_v1_servicemonitor_${OPERATOR_NAME}-controller-manager-metrics-monitor.yaml
-	helm lint ./charts/${OPERATOR_NAME}	
+	$(HELM) lint ./charts/${OPERATOR_NAME}	
 
 helmchart-repo: helmchart
 	mkdir -p ${HELM_REPO_DEST}/${OPERATOR_NAME}
-	helm package -d ${HELM_REPO_DEST}/${OPERATOR_NAME} ./charts/${OPERATOR_NAME}
-	helm repo index --url ${CHART_REPO_URL} ${HELM_REPO_DEST}
+	$(HELM) package -d ${HELM_REPO_DEST}/${OPERATOR_NAME} ./charts/${OPERATOR_NAME}
+	$(HELM) repo index --url ${CHART_REPO_URL} ${HELM_REPO_DEST}
 
 helmchart-repo-push: helmchart-repo	
 	git -C ${HELM_REPO_DEST} add .
 	git -C ${HELM_REPO_DEST} status
 	git -C ${HELM_REPO_DEST} commit -m "Release ${VERSION}"
 	git -C ${HELM_REPO_DEST} push origin "gh-pages"	
+
+HELM_TEST_IMG_NAME ?= ${OPERATOR_NAME}
+HELM_TEST_IMG_TAG ?= helmchart-test
+
+# Deploy the helmchart to a kind cluster to test deployment.
+# If the test-metrics sidecar in the prometheus pod is ready, the metrics work and the test is successful.
+.PHONY: helmchart-test
+helmchart-test: kind-setup helmchart
+	$(MAKE) IMG=${HELM_TEST_IMG_NAME}:${HELM_TEST_IMG_TAG} docker-build
+	docker tag ${HELM_TEST_IMG_NAME}:${HELM_TEST_IMG_TAG} docker.io/library/${HELM_TEST_IMG_NAME}:${HELM_TEST_IMG_TAG}
+	$(KIND) load docker-image ${HELM_TEST_IMG_NAME}:${HELM_TEST_IMG_TAG} docker.io/library/${HELM_TEST_IMG_NAME}:${HELM_TEST_IMG_TAG}
+	$(HELM) repo add jetstack https://charts.jetstack.io
+	$(HELM) install cert-manager jetstack/cert-manager --namespace cert-manager --create-namespace --version v1.7.1 --set installCRDs=true
+	$(HELM) repo add prometheus-community https://prometheus-community.github.io/helm-charts
+	$(HELM) install kube-prometheus-stack prometheus-community/kube-prometheus-stack -n default -f integration/kube-prometheus-stack-values.yaml
+	$(HELM) install prometheus-rbac integration/helm/prometheus-rbac -n default
+	$(HELM) upgrade -i ${OPERATOR_NAME}-local charts/${OPERATOR_NAME} -n ${OPERATOR_NAME}-local --create-namespace \
+	  --set enableCertManager=true \
+	  --set image.repository=${HELM_TEST_IMG_NAME} \
+	  --set image.tag=${HELM_TEST_IMG_TAG}
+	$(KUBECTL) wait --namespace ${OPERATOR_NAME}-local --for=condition=ready pod --selector=app.kubernetes.io/name=${OPERATOR_NAME} --timeout=90s
+	$(KUBECTL) wait --namespace default --for=condition=ready pod prometheus-kube-prometheus-stack-prometheus-0 --timeout=180s
+	$(KUBECTL) exec prometheus-kube-prometheus-stack-prometheus-0 -n default -c test-metrics -- /bin/sh -c "echo 'Example metrics...' && cat /tmp/ready"
+
+.PHONY: kind
+KIND = ./bin/kind
+kind: ## Download kind locally if necessary.
+ifeq (,$(wildcard $(KIND)))
+ifeq (,$(shell which kind 2>/dev/null))
+	$(call go-get-tool,$(KIND),sigs.k8s.io/kind@${KIND_VERSION})
+else
+KIND = $(shell which kind)
+endif
+endif
+
+.PHONY: kubectl
+KUBECTL = ./bin/kubectl
+kubectl: ## Download kubectl locally if necessary.
+ifeq (,$(wildcard $(KUBECTL)))
+ifeq (,$(shell which kubectl 2>/dev/null))
+	echo "Downloading ${KUBECTL} for managing k8s resources."
+	OS=$(shell go env GOOS) ;\
+	ARCH=$(shell go env GOARCH) ;\
+	curl --create-dirs -sSLo ${KUBECTL} https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/$${OS}/$${ARCH}/kubectl ;\
+	chmod +x ${KUBECTL}
+else
+KUBECTL = $(shell which kubectl)
+endif
+endif
+
+.PHONY: helm
+HELM = ./bin/helm
+helm: ## Download helm locally if necessary.
+ifeq (,$(wildcard $(HELM)))
+ifeq (,$(shell which helm 2>/dev/null))
+	echo "Downloading ${HELM}."
+	OS=$(shell go env GOOS) ;\
+	ARCH=$(shell go env GOARCH) ;\
+	curl --create-dirs -sSLo ${HELM}.tar.gz https://get.helm.sh/helm-${HELM_VERSION}-$${OS}-$${ARCH}.tar.gz ;\
+	tar -xf ${HELM}.tar.gz -C ./bin/ ;\
+	mv ./bin/$${OS}-$${ARCH}/helm ${HELM}
+else
+HELM = $(shell which helm)
+endif
+endif

config/helmchart/cert-manager-ca-injection.yaml

@@ -1,4 +1,4 @@
 - op: add
   path: /metadata/annotations
   value:
-    cert-manager.io/inject-ca-from: "{{ .Release.Namespace }}/webhook-server-cert"
+    cert-manager.io/inject-ca-from: "{{ .Release.Namespace }}/serving-cert"

config/helmchart/templates/certificate.yaml

@@ -18,4 +18,17 @@ spec:
     kind: Issuer
     name: selfsigned-issuer
   secretName: webhook-server-cert
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: metrics-serving-cert
+spec:
+  dnsNames:
+  - patch-operator-controller-manager-metrics-service.{{ .Release.Namespace }}.svc
+  - patch-operator-controller-manager-metrics-service.{{ .Release.Namespace }}.svc.cluster.local
+  issuerRef:
+    kind: Issuer
+    name: selfsigned-issuer
+  secretName: patch-operator-certs  
 {{ end }}

config/helmchart/templates/manager.yaml

@@ -8,6 +8,7 @@ spec:
   selector:
     matchLabels:
       {{- include "patch-operator.selectorLabels" . | nindent 6 }}
+      control-plane: patch-operator
   replicas: {{ .Values.replicaCount }}
   template:
     metadata:
@@ -17,7 +18,7 @@ spec:
     {{- end }}
       labels:
         {{- include "patch-operator.selectorLabels" . | nindent 8 }}
-        operator: patch-operator
+        control-plane: patch-operator
     spec:
       serviceAccountName: controller-manager
       {{- with .Values.imagePullSecrets }}
@@ -39,7 +40,7 @@ spec:
           name: https
         volumeMounts:
         - mountPath: /etc/certs/tls
-          name: tls-cert
+          name: patch-operator-certs
         imagePullPolicy: {{ .Values.kube_rbac_proxy.image.pullPolicy }}
         resources:
           {{- toYaml .Values.kube_rbac_proxy.resources | nindent 10 }}
@@ -50,7 +51,7 @@ spec:
         image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
         imagePullPolicy: {{ .Values.image.pullPolicy }}
         volumeMounts:
-        - name: cert
+        - name: webhook-server-cert
           readOnly: true
           mountPath: /tmp/k8s-webhook-server/serving-certs
         {{- with .Values.env }}
@@ -85,11 +86,11 @@ spec:
         {{- toYaml . | nindent 8 }}
       {{- end }}
       volumes:
-      - name: tls-cert
+      - name: patch-operator-certs
         secret:
           defaultMode: 420
           secretName: patch-operator-certs  
-      - name: cert
+      - name: webhook-server-cert
         secret:
           secretName: webhook-server-cert
           defaultMode: 420              

config/manager/manager.yaml

@@ -2,7 +2,7 @@ apiVersion: v1
 kind: Namespace
 metadata:
   labels:
-    control-plane: controller-manager
+    control-plane: patch-operator
     openshift.io/cluster-monitoring: "true"
   name: system
 ---
@@ -12,16 +12,16 @@ metadata:
   name: controller-manager
   namespace: system
   labels:
-    control-plane: controller-manager
+    control-plane: patch-operator
 spec:
   selector:
     matchLabels:
-      control-plane: controller-manager
+      control-plane: patch-operator
   replicas: 1
   template:
     metadata:
       labels:
-        control-plane: controller-manager
+        control-plane: patch-operator
     spec:
       securityContext:
         runAsNonRoot: true
@@ -38,12 +38,14 @@ spec:
           httpGet:
             path: /healthz
             port: 8081
+            scheme: HTTP
           initialDelaySeconds: 15
           periodSeconds: 20
         readinessProbe:
           httpGet:
             path: /readyz
             port: 8081
+            scheme: HTTP
           initialDelaySeconds: 5
           periodSeconds: 10
         resources: