Skip to content

Commit e0d3fe8

Browse files
IshwarKanseTylerHelmuth
IshwarKanse
and
TylerHelmuth
authored
Add functional test for OTLP metrics collection and forward to prometheus (#1850)
Signed-off-by: Ishwar Kanse <[email protected]> Co-authored-by: Tyler Helmuth <[email protected]>
1 parent 92e5dee commit e0d3fe8

17 files changed

+293
-0
lines changed

‎CONTRIBUTING.md

+9
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,15 @@ Once they are installed, the tests can be executed with `make prepare-e2e`, whic
107107

108108
The tests are located under `tests/e2e` and are written to be used with `kuttl`. Refer to their documentation to understand how tests are written.
109109

110+
### OpenShift End to End tests
111+
To run the end-to-end tests written for OpenShift, you'll need a OpenShift cluster.
112+
113+
To install the OpenTelemetry operator, please follow the instructions in [Operator Lifecycle Manager (OLM)](https://github.com/open-telemetry/opentelemetry-operator/blob/main/CONTRIBUTING.md#operator-lifecycle-manager-olm)
114+
115+
Once the operator is installed, the tests can be executed using `make e2e-openshift`, which will call to the `e2e-openshift` target. Note that `kind` is disabled for the TestSuite as the requirement is to use an OpenShift cluster for these test cases.
116+
117+
The tests are located under `tests/e2e-openshift` and are written to be used with `kuttl`.
118+
110119
### Undeploying the operator from the local cluster
111120

112121
```bash

‎Makefile

+5
Original file line numberDiff line numberDiff line change
@@ -186,6 +186,11 @@ e2e-upgrade: undeploy
186186
e2e-autoscale:
187187
$(KUTTL) test --config kuttl-test-autoscale.yaml
188188

189+
# end-to-end-test for testing OpenShift cases
190+
.PHONY: e2e-openshift
191+
e2e-openshift:
192+
$(KUTTL) test --config kuttl-test-openshift.yaml
193+
189194
.PHONY: e2e-log-operator
190195
e2e-log-operator:
191196
kubectl get pod -n opentelemetry-operator-system | grep "opentelemetry-operator" | awk '{print $$1}' | xargs -I {} kubectl logs -n opentelemetry-operator-system {} manager

‎kuttl-test-openshift.yaml

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
apiVersion: kuttl.dev/v1beta1
2+
kind: TestSuite
3+
startKIND: false
4+
testDirs:
5+
- ./tests/e2e-openshift/
6+
timeout: 150

‎tests/e2e-openshift/otlp-metrics-traces/00-assert.yaml

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
---
2+
apiVersion: apps/v1
3+
kind: Deployment
4+
metadata:
5+
namespace: kuttl-otlp-metrics
6+
name: jaeger-allinone
7+
status:
8+
availableReplicas: 1
9+
readyReplicas: 1
10+
replicas: 1

‎tests/e2e-openshift/otlp-metrics-traces/00-install-jaeger.yaml

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
#Create the "kuttl-otlp-metrics" project to fulfill the requirement of specifying the Jaeger and OTEL collector endpoints. This project is essential due to KUTTL's current lack of support for templating.
2+
#For this test case you'll need to install the Jaeger operator (OpenShift Distributed Tracing Platform in OpenShift)
3+
4+
apiVersion: v1
5+
kind: Namespace
6+
metadata:
7+
name: kuttl-otlp-metrics
8+
9+
---
10+
apiVersion: jaegertracing.io/v1
11+
kind: Jaeger
12+
metadata:
13+
name: jaeger-allinone
14+
namespace: kuttl-otlp-metrics
15+
spec:
16+
strategy: allinone
17+
ingress:
18+
security: none

‎tests/e2e-openshift/otlp-metrics-traces/01-assert.yaml

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
---
2+
apiVersion: apps/v1
3+
kind: Deployment
4+
metadata:
5+
namespace: openshift-user-workload-monitoring
6+
status:
7+
availableReplicas: 1
8+
readyReplicas: 1
9+
replicas: 1
10+
11+
---
12+
apiVersion: apps/v1
13+
kind: StatefulSet
14+
metadata:
15+
namespace: openshift-user-workload-monitoring
16+
status:
17+
availableReplicas: 2
18+
currentReplicas: 2
19+
readyReplicas: 2
20+
replicas: 2
21+
updatedReplicas: 2
22+

‎tests/e2e-openshift/otlp-metrics-traces/01-workload-monitoring.yaml

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
# oc -n openshift-user-workload-monitoring get pod
2+
# https://docs.openshift.com/container-platform/4.13/monitoring/enabling-monitoring-for-user-defined-projects.html#accessing-metrics-from-outside-cluster_enabling-monitoring-for-user-defined-projects
3+
4+
apiVersion: v1
5+
kind: ConfigMap
6+
metadata:
7+
name: cluster-monitoring-config
8+
namespace: openshift-monitoring
9+
data:
10+
config.yaml: |
11+
enableUserWorkload: true

‎tests/e2e-openshift/otlp-metrics-traces/02-assert.yaml

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
---
2+
apiVersion: apps/v1
3+
kind: Deployment
4+
metadata:
5+
namespace: kuttl-otlp-metrics
6+
status:
7+
availableReplicas: 1
8+
readyReplicas: 1
9+
replicas: 1

‎tests/e2e-openshift/otlp-metrics-traces/02-otel-metrics-collector.yaml

+39
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
#https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/exporter/prometheusexporter/README.md
2+
3+
apiVersion: opentelemetry.io/v1alpha1
4+
kind: OpenTelemetryCollector
5+
metadata:
6+
name: cluster-collector
7+
namespace: kuttl-otlp-metrics
8+
spec:
9+
mode: deployment
10+
ports:
11+
- name: promexporter
12+
port: 8889
13+
protocol: TCP
14+
config: |
15+
receivers:
16+
otlp:
17+
protocols:
18+
grpc:
19+
http:
20+
processors:
21+
exporters:
22+
jaeger:
23+
endpoint: jaeger-allinone-collector-headless.kuttl-otlp-metrics.svc:14250
24+
tls:
25+
ca_file: "/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt"
26+
prometheus:
27+
endpoint: 0.0.0.0:8889
28+
resource_to_telemetry_conversion:
29+
enabled: true # by default resource attributes are dropped
30+
service:
31+
pipelines:
32+
traces:
33+
receivers: [otlp]
34+
processors: []
35+
exporters: [jaeger]
36+
metrics:
37+
receivers: [otlp]
38+
processors: []
39+
exporters: [prometheus]

‎tests/e2e-openshift/otlp-metrics-traces/03-assert.yaml

+25
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
---
2+
apiVersion: monitoring.coreos.com/v1
3+
kind: PodMonitor
4+
metadata:
5+
name: otel-collector
6+
namespace: kuttl-otlp-metrics
7+
spec:
8+
podMetricsEndpoints:
9+
- port: metrics
10+
- metricRelabelings:
11+
- action: labeldrop
12+
regex: instance
13+
- action: labeldrop
14+
regex: job
15+
port: promexporter
16+
relabelings:
17+
- action: labeldrop
18+
regex: pod
19+
- action: labeldrop
20+
regex: container
21+
- action: labeldrop
22+
regex: endpoint
23+
selector:
24+
matchLabels:
25+
app.kubernetes.io/name: cluster-collector-collector

‎tests/e2e-openshift/otlp-metrics-traces/03-podmonitor.yaml

+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
#https://docs.openshift.com/container-platform/4.13/monitoring/managing-metrics.html
2+
#https://docs.openshift.com/container-platform/4.13/rest_api/monitoring_apis/podmonitor-monitoring-coreos-com-v1.html
3+
apiVersion: monitoring.coreos.com/v1
4+
kind: PodMonitor
5+
metadata:
6+
name: otel-collector
7+
namespace: kuttl-otlp-metrics
8+
spec:
9+
selector:
10+
matchLabels:
11+
app.kubernetes.io/name: cluster-collector-collector
12+
podMetricsEndpoints:
13+
- port: metrics
14+
- port: promexporter
15+
relabelings:
16+
- action: labeldrop
17+
regex: pod
18+
- action: labeldrop
19+
regex: container
20+
- action: labeldrop
21+
regex: endpoint
22+
metricRelabelings:
23+
- action: labeldrop
24+
regex: instance
25+
- action: labeldrop
26+
regex: job

‎tests/e2e-openshift/otlp-metrics-traces/04-assert.yaml

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
apiVersion: batch/v1
2+
kind: Job
3+
metadata:
4+
name: telemetrygen-traces
5+
status:
6+
active: 1
7+
ready: 1
8+
9+
---
10+
apiVersion: batch/v1
11+
kind: Job
12+
metadata:
13+
name: telemetrygen-metrics
14+
status:
15+
active: 1
16+
ready: 1
17+
18+
---
19+
apiVersion: v1
20+
kind: Pod
21+
status:
22+
phase: Running

‎tests/e2e-openshift/otlp-metrics-traces/04-metrics-traces-gen.yaml

+52
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
apiVersion: batch/v1
2+
kind: Job
3+
metadata:
4+
name: telemetrygen-traces
5+
spec:
6+
completions: 1
7+
parallelism: 1
8+
template:
9+
metadata:
10+
labels:
11+
app: telemetrygen-traces
12+
spec:
13+
containers:
14+
- name: telemetrygen-traces
15+
image: ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:latest
16+
command: ["./telemetrygen"]
17+
args:
18+
- "--otlp-endpoint=cluster-collector-collector-headless.kuttl-otlp-metrics.svc:4317"
19+
- "--otlp-insecure=true"
20+
- "--rate=1"
21+
- "--duration=30s"
22+
- "--otlp-attributes=telemetrygen=\"traces\""
23+
- "--otlp-header=telemetrygen=\"traces\""
24+
- "traces"
25+
restartPolicy: Never
26+
27+
---
28+
apiVersion: batch/v1
29+
kind: Job
30+
metadata:
31+
name: telemetrygen-metrics
32+
spec:
33+
completions: 1
34+
parallelism: 1
35+
template:
36+
metadata:
37+
labels:
38+
app: telemetrygen-metrics
39+
spec:
40+
containers:
41+
- name: telemetrygen-metrics
42+
image: ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:latest
43+
command: ["./telemetrygen"]
44+
args:
45+
- "--otlp-endpoint=cluster-collector-collector-headless.kuttl-otlp-metrics.svc:4317"
46+
- "--otlp-insecure=true"
47+
- "--duration=30s"
48+
- "--rate=1"
49+
- "--otlp-attributes=telemetrygen=\"metrics\""
50+
- "--otlp-header=telemetrygen=\"traces\""
51+
- "metrics"
52+
restartPolicy: Never

‎tests/e2e-openshift/otlp-metrics-traces/05-assert.yaml

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
---
2+
apiVersion: kuttl.dev/v1beta1
3+
kind: TestAssert
4+
timeout: 30
5+
commands:
6+
- script: ./tests/e2e-openshift/otlp-metrics/check_traces.sh

‎tests/e2e-openshift/otlp-metrics-traces/06-assert.yaml

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
---
2+
apiVersion: kuttl.dev/v1beta1
3+
kind: TestAssert
4+
timeout: 30
5+
commands:
6+
- script: ./tests/e2e-openshift/otlp-metrics/check_metrics.sh

‎tests/e2e-openshift/otlp-metrics-traces/check_metrics.sh

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
#!/bin/bash
2+
SECRET=$(oc get secret -n openshift-user-workload-monitoring | grep prometheus-user-workload-token | head -n 1 | awk '{print $1}')
3+
TOKEN=$(echo $(oc get secret $SECRET -n openshift-user-workload-monitoring -o json | jq -r '.data.token') | base64 -d)
4+
THANOS_QUERIER_HOST=$(oc get route thanos-querier -n openshift-monitoring -o json | jq -r '.spec.host')
5+
6+
response=$(curl -k -H "Authorization: Bearer $TOKEN" -H "Content-type: application/json" "https://$THANOS_QUERIER_HOST/api/v1/query?query=gen")
7+
8+
count=$(echo "$response" | jq -r '.data.result | length')
9+
10+
if [[ $count -eq 0 ]]; then
11+
echo "No telemetrygen metrics count with value present. Exiting with status 1."
12+
exit 1
13+
else
14+
echo "telemetrygen metrics with value is present."
15+
fi

‎tests/e2e-openshift/otlp-metrics-traces/check_traces.sh

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
#!/bin/bash
2+
JAEGER_URL=$(oc -n kuttl-otlp-metrics get route jaeger-allinone -o json | jq '.spec.host' -r)
3+
SERVICE_NAME="telemetrygen"
4+
5+
trace_exists=$(curl -ksSL "https://$JAEGER_URL/api/traces?service=$SERVICE_NAME&limit=1" | jq -r '.data | length')
6+
7+
if [[ $trace_exists -gt 0 ]]; then
8+
echo "Traces for $SERVICE_NAME exist in Jaeger."
9+
else
10+
echo "Trace for $SERVICE_NAME does not exist in Jaeger."
11+
exit 1 # Fail the test step if the trace doesn't exist
12+
fi

0 commit comments

Comments
 (0)