Add E2E test; put TA E2E tests into separate target

Signed-off-by: Matej Gera <[email protected]>

Author: matej-g

Makefile

@@ -233,6 +233,11 @@ e2e-multi-instrumentation:
 e2e-opampbridge:
 	$(KUTTL) test --config kuttl-test-opampbridge.yaml
 
+# Target allocator end-to-tests
+.PHONY: e2e-targetallocator
+e2e-targetallocator:
+	$(KUTTL) test --config kuttl-test-targetallocator.yaml
+
 .PHONY: prepare-e2e
 prepare-e2e: kuttl set-image-controller container container-target-allocator container-operator-opamp-bridge start-kind cert-manager install-metrics-server install-targetallocator-prometheus-crds load-image-all deploy
 	TARGETALLOCATOR_IMG=$(TARGETALLOCATOR_IMG) OPERATOROPAMPBRIDGE_IMG=$(OPERATOROPAMPBRIDGE_IMG) OPERATOR_IMG=$(IMG) SED_BIN="$(SED)" ./hack/modify-test-images.sh

hack/modify-test-images.sh

@@ -7,9 +7,10 @@ DEFAULT_OPERATOROPAMPBRIDGE_IMG=${DEFAULT_OPERATOROPAMPBRIDGE_IMG:-local/opentel
 DEFAULT_OPERATOR_IMG=${DEFAULT_OPERATOR_IMG:-local/opentelemetry-operator:e2e}
 
 ${SED_BIN} -i "s#${DEFAULT_TARGETALLOCATOR_IMG}#${TARGETALLOCATOR_IMG}#g" tests/e2e/smoke-targetallocator/*.yaml
-${SED_BIN} -i "s#${DEFAULT_TARGETALLOCATOR_IMG}#${TARGETALLOCATOR_IMG}#g" tests/e2e/targetallocator-features/00-install.yaml
+${SED_BIN} -i "s#${DEFAULT_TARGETALLOCATOR_IMG}#${TARGETALLOCATOR_IMG}#g" tests/e2e-targetallocator/targetallocator-features/00-install.yaml
 ${SED_BIN} -i "s#${DEFAULT_TARGETALLOCATOR_IMG}#${TARGETALLOCATOR_IMG}#g" tests/e2e/prometheus-config-validation/*.yaml
-${SED_BIN} -i "s#${DEFAULT_TARGETALLOCATOR_IMG}#${TARGETALLOCATOR_IMG}#g" tests/e2e/targetallocator-prometheuscr/*.yaml
+${SED_BIN} -i "s#${DEFAULT_TARGETALLOCATOR_IMG}#${TARGETALLOCATOR_IMG}#g" tests/e2e-targetallocator/targetallocator-prometheuscr/*.yaml
+${SED_BIN} -i "s#${DEFAULT_TARGETALLOCATOR_IMG}#${TARGETALLOCATOR_IMG}#g" tests/e2e-targetallocator/targetallocator-kubernetessd/*.yaml
 
 ${SED_BIN} -i "s#${DEFAULT_OPERATOR_IMG}#${OPERATOR_IMG}#g" tests/e2e-multi-instrumentation/*.yaml
 

kuttl-test-targetallocator.yaml

@@ -0,0 +1,7 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestSuite
+artifactsDir: ./tests/_build/artifacts/
+testDirs:
+  - ./tests/e2e-targetallocator/
+timeout: 150
+parallel: 4

tests/e2e-opampbridge/opampbridge/00-install.yaml

@@ -26,7 +26,7 @@ kind: OpAMPBridge
 metadata:
   name: test
 spec:
-  image: "local/opentelemetry-operator-opamp-bridge:e2e"
+  image: "ghcr.io/open-telemetry/opentelemetry-operator/operator-opamp-bridge:v0.90.0-14-g082e678"
   endpoint: ws://opamp-server:4320/v1/opamp
   capabilities:
     AcceptsOpAMPConnectionSettings: true

tests/e2e/targetallocator-features/00-assert.yaml tests/e2e-targetallocator/targetallocator-features/00-assert.yaml

@@ -0,0 +1,52 @@
+# This KUTTL assert uses the check-daemonset.sh script to ensure the number of ready pods in a daemonset matches the desired count, retrying until successful or a timeout occurs. The script is needed as the number of Kubernetes cluster nodes can vary and we cannot statically set desiredNumberScheduled and numberReady in the assert for daemonset status. 
+
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+commands:
+- script: ./tests/e2e/smoke-daemonset/check-daemonset.sh
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: prometheus-kubernetessd-targetallocator
+status:
+  replicas: 1
+  readyReplicas: 1
+  observedGeneration: 1
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: prometheus-kubernetessd-targetallocator
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: prometheus-kubernetessd-collector
+data:
+  collector.yaml: |
+    exporters:
+      prometheus:
+        endpoint: 0.0.0.0:9090
+    processors: null
+    receivers:
+      prometheus:
+        config: {}
+        target_allocator:
+          collector_id: ${POD_NAME}
+          endpoint: http://prometheus-kubernetessd-targetallocator:80
+          interval: 30s
+    service:
+      pipelines:
+        metrics:
+          exporters:
+          - prometheus
+          processors: []
+          receivers:
+          - prometheus
+---
+# Print TA pod logs if test fails
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+collectors:
+  - selector: app.kubernetes.io/managed-by=opentelemetry-operator

tests/e2e/targetallocator-features/00-install.yaml tests/e2e-targetallocator/targetallocator-features/00-install.yaml

@@ -0,0 +1,133 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: ta
+automountServiceAccountToken: true
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: collector
+automountServiceAccountToken: true
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: ta
+rules:
+- apiGroups: [""]
+  resources:
+    - pods
+    - nodes
+    - services
+    - endpoints
+    - configmaps
+    - secrets
+    - namespaces
+  verbs:
+    - get
+    - watch
+    - list
+- apiGroups: ["apps"]
+  resources:
+    - statefulsets
+    - daemonsets
+    - services
+    - endpoints
+  verbs:
+    - get
+    - watch
+    - list
+- apiGroups: ["discovery.k8s.io"]
+  resources:
+    - endpointslices
+  verbs:
+    - get
+    - watch
+    - list
+- apiGroups: ["networking.k8s.io"]
+  resources:
+    - ingresses
+  verbs:
+    - get
+    - watch
+    - list
+- nonResourceURLs: ["/metrics"]
+  verbs: ["get"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: collector
+rules:
+- apiGroups: [""]
+  resources:
+    - pods
+    - nodes
+    - nodes/metrics
+    - services
+    - endpoints
+  verbs:
+    - get
+    - watch
+    - list
+- apiGroups: ["networking.k8s.io"]
+  resources:
+    - ingresses
+  verbs:
+    - get
+    - watch
+    - list
+- nonResourceURLs: ["/metrics", "/metrics/cadvisor"]
+  verbs: ["get"]
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - command: kubectl create clusterrolebinding ta-$NAMESPACE --clusterrole=ta --serviceaccount=$NAMESPACE:ta
+  - command: kubectl create clusterrolebinding collector-$NAMESPACE --clusterrole=collector --serviceaccount=$NAMESPACE:collector
+---
+apiVersion: opentelemetry.io/v1alpha1
+kind: OpenTelemetryCollector
+metadata:
+  name: prometheus-kubernetessd
+spec:
+  mode: daemonset
+  serviceAccount: collector
+  targetAllocator:
+    enabled: true
+    serviceAccount: ta
+    image: "local/opentelemetry-operator-targetallocator:e2e"
+    prometheusCR:
+      enabled: false
+  config: |
+    receivers:
+      prometheus:
+        config:
+          scrape_configs:
+          - job_name: kubelet
+            scheme: https
+            authorization:
+              credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+            tls_config:
+              ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+              insecure_skip_verify: true
+            honor_labels: true
+            kubernetes_sd_configs:
+              - role: node
+            metric_relabel_configs:
+              - action: keep
+                regex: "kubelet_running_pods"
+                source_labels: [__name__]
+
+    processors:
+
+    exporters:
+      prometheus:
+        endpoint: 0.0.0.0:9090
+    service:
+      pipelines:
+        metrics:
+          receivers: [prometheus]
+          processors: []
+          exporters: [prometheus]

tests/e2e/targetallocator-features/01-assert.yaml tests/e2e-targetallocator/targetallocator-features/01-assert.yaml

@@ -0,0 +1,49 @@
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: check-metrics
+spec:
+  template:
+    spec:
+      restartPolicy: OnFailure
+      containers:
+        - name: check-metrics
+          image: curlimages/curl
+          args:
+            - /bin/sh
+            - -c
+            - curl -s http://prometheus-kubernetessd-collector:9090/metrics | grep "kubelet_running_pods{"
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: check-ta-jobs
+spec:
+  template:
+    spec:
+      restartPolicy: OnFailure
+      containers:
+        - name: check-metrics
+          image: curlimages/curl
+          args:
+            - /bin/sh
+            - -c
+            - curl -s http://prometheus-kubernetessd-targetallocator/scrape_configs | grep "kubelet"
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: check-ta-scrape-configs
+spec:
+  template:
+    spec:
+      restartPolicy: OnFailure
+      containers:
+        - name: check-metrics
+          image: curlimages/curl
+          args:
+            - /bin/sh
+            - -c
+            # First get the collector pod name, subsequently check that the targets for the collector include the node name label.
+            - curl -s http://prometheus-kubernetessd-targetallocator/jobs/kubelet/targets?collector_id=$(curl -s http://prometheus-kubernetessd-targetallocator/jobs/kubelet/targets | grep -oE "prometheus-kubernetessd-collector-.{5}") | grep "__meta_kubernetes_node_name"

tests/e2e/targetallocator-features/01-liveness.yaml tests/e2e-targetallocator/targetallocator-features/01-liveness.yaml

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+# Name of the daemonset to check
+DAEMONSET_NAME="prometheus-kubernetessd-collector"
+
+# Get the desired and ready pod counts for the daemonset
+read DESIRED READY <<< $(kubectl get daemonset -n $NAMESPACE $DAEMONSET_NAME -o custom-columns=:status.desiredNumberScheduled,:status.numberReady --no-headers)
+
+# Check if the desired count matches the ready count
+if [ "$DESIRED" -eq "$READY" ]; then
+  echo "Desired count ($DESIRED) matches the ready count ($READY) for $DAEMONSET_NAME."
+else
+  echo "Desired count ($DESIRED) does not match the ready count ($READY) for $DAEMONSET_NAME."
+  exit 1
+fi

tests/e2e-targetallocator/targetallocator-kubernetessd/00-assert.yaml

@@ -0,0 +1,20 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: check-metrics
+status:
+  succeeded: 1
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: check-ta-jobs
+status:
+  succeeded: 1
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: check-ta-scrape-configs
+status:
+  succeeded: 1