From 52442cfb6163f3e7c8d190152623930a0dac369c Mon Sep 17 00:00:00 2001 From: Jacob Aronoff Date: Wed, 3 Jul 2024 10:23:15 -0400 Subject: [PATCH 1/2] update e2e test --- .../opampbridge/00-assert.yaml | 62 +------------ .../opampbridge/00-install.yaml | 86 +++++++++---------- .../opampbridge/01-assert.yaml | 65 ++++++++++++++ .../opampbridge/01-install.yaml | 50 +++++++++++ .../opampbridge/chainsaw-test.yaml | 12 ++- 5 files changed, 167 insertions(+), 108 deletions(-) create mode 100644 tests/e2e-opampbridge/opampbridge/01-assert.yaml create mode 100644 tests/e2e-opampbridge/opampbridge/01-install.yaml diff --git a/tests/e2e-opampbridge/opampbridge/00-assert.yaml b/tests/e2e-opampbridge/opampbridge/00-assert.yaml index bc576139e3..e0b6f37b83 100644 --- a/tests/e2e-opampbridge/opampbridge/00-assert.yaml +++ b/tests/e2e-opampbridge/opampbridge/00-assert.yaml @@ -1,65 +1,7 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: test-opamp-bridge -spec: - template: - spec: - containers: - - env: - - name: OTELCOL_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - name: opamp-bridge-container - volumeMounts: - - mountPath: /conf - name: opamp-bridge-internal - volumes: - - configMap: - items: - - key: remoteconfiguration.yaml - path: remoteconfiguration.yaml - name: test-opamp-bridge - name: opamp-bridge-internal + name: e2e-test-app-bridge-server status: readyReplicas: 1 - replicas: 1 ---- -apiVersion: v1 -data: - remoteconfiguration.yaml: | - capabilities: - AcceptsOpAMPConnectionSettings: true - AcceptsOtherConnectionSettings: true - AcceptsRemoteConfig: true - AcceptsRestartCommand: true - ReportsEffectiveConfig: true - ReportsHealth: true - ReportsOwnLogs: true - ReportsOwnMetrics: true - ReportsOwnTraces: true - ReportsRemoteConfig: true - ReportsStatus: true - componentsAllowed: - exporters: - - logging - processors: - - memory_limiter - receivers: - - otlp - endpoint: ws://opamp-server:4320/v1/opamp -kind: ConfigMap -metadata: - name: test-opamp-bridge ---- -apiVersion: v1 -kind: Service -metadata: - name: test-opamp-bridge -spec: - ports: - - name: opamp-bridge - port: 80 - protocol: TCP - targetPort: 8080 + replicas: 1 \ No newline at end of file diff --git a/tests/e2e-opampbridge/opampbridge/00-install.yaml b/tests/e2e-opampbridge/opampbridge/00-install.yaml index f76946e125..d06a21c43c 100644 --- a/tests/e2e-opampbridge/opampbridge/00-install.yaml +++ b/tests/e2e-opampbridge/opampbridge/00-install.yaml @@ -1,50 +1,44 @@ -apiVersion: v1 -automountServiceAccountToken: true -kind: ServiceAccount -metadata: - name: opamp-bridge ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role +apiVersion: apps/v1 +kind: Deployment metadata: - name: opamp-bridge -rules: -- apiGroups: - - opentelemetry.io - resources: - - opentelemetrycollectors - verbs: - - '*' -- apiGroups: - - "" - resources: - - pods - verbs: - - list - - get + name: e2e-test-app-bridge-server +spec: + replicas: 1 + selector: + matchLabels: + app: e2e-test-app-bridge-server + template: + metadata: + labels: + app: e2e-test-app-bridge-server + spec: + containers: + - name: e2e-test-app-bridge-server + image: ghcr.io/open-telemetry/opentelemetry-operator/e2e-test-app-bridge-server:main + ports: + - containerPort: 4320 + - containerPort: 4321 + resources: + limits: + memory: "128Mi" + cpu: "250m" + requests: + memory: "64Mi" + cpu: "100m" --- -apiVersion: opentelemetry.io/v1alpha1 -kind: OpAMPBridge +apiVersion: v1 +kind: Service metadata: - name: test + name: e2e-test-app-bridge-server spec: - capabilities: - AcceptsOpAMPConnectionSettings: true - AcceptsOtherConnectionSettings: true - AcceptsRemoteConfig: true - AcceptsRestartCommand: true - ReportsEffectiveConfig: true - ReportsHealth: true - ReportsOwnLogs: true - ReportsOwnMetrics: true - ReportsOwnTraces: true - ReportsRemoteConfig: true - ReportsStatus: true - componentsAllowed: - exporters: - - logging - processors: - - memory_limiter - receivers: - - otlp - endpoint: ws://opamp-server:4320/v1/opamp + selector: + app: e2e-test-app-bridge-server + ports: + - protocol: TCP + port: 4320 + targetPort: 4320 + name: "opamp" + - protocol: TCP + port: 4321 + targetPort: 4321 + name: "http" diff --git a/tests/e2e-opampbridge/opampbridge/01-assert.yaml b/tests/e2e-opampbridge/opampbridge/01-assert.yaml new file mode 100644 index 0000000000..bc576139e3 --- /dev/null +++ b/tests/e2e-opampbridge/opampbridge/01-assert.yaml @@ -0,0 +1,65 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-opamp-bridge +spec: + template: + spec: + containers: + - env: + - name: OTELCOL_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + name: opamp-bridge-container + volumeMounts: + - mountPath: /conf + name: opamp-bridge-internal + volumes: + - configMap: + items: + - key: remoteconfiguration.yaml + path: remoteconfiguration.yaml + name: test-opamp-bridge + name: opamp-bridge-internal +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: v1 +data: + remoteconfiguration.yaml: | + capabilities: + AcceptsOpAMPConnectionSettings: true + AcceptsOtherConnectionSettings: true + AcceptsRemoteConfig: true + AcceptsRestartCommand: true + ReportsEffectiveConfig: true + ReportsHealth: true + ReportsOwnLogs: true + ReportsOwnMetrics: true + ReportsOwnTraces: true + ReportsRemoteConfig: true + ReportsStatus: true + componentsAllowed: + exporters: + - logging + processors: + - memory_limiter + receivers: + - otlp + endpoint: ws://opamp-server:4320/v1/opamp +kind: ConfigMap +metadata: + name: test-opamp-bridge +--- +apiVersion: v1 +kind: Service +metadata: + name: test-opamp-bridge +spec: + ports: + - name: opamp-bridge + port: 80 + protocol: TCP + targetPort: 8080 diff --git a/tests/e2e-opampbridge/opampbridge/01-install.yaml b/tests/e2e-opampbridge/opampbridge/01-install.yaml new file mode 100644 index 0000000000..2d5892df85 --- /dev/null +++ b/tests/e2e-opampbridge/opampbridge/01-install.yaml @@ -0,0 +1,50 @@ +apiVersion: v1 +automountServiceAccountToken: true +kind: ServiceAccount +metadata: + name: opamp-bridge +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: opamp-bridge +rules: + - apiGroups: + - opentelemetry.io + resources: + - opentelemetrycollectors + verbs: + - '*' + - apiGroups: + - "" + resources: + - pods + verbs: + - list + - get +--- +apiVersion: opentelemetry.io/v1alpha1 +kind: OpAMPBridge +metadata: + name: test +spec: + capabilities: + AcceptsOpAMPConnectionSettings: true + AcceptsOtherConnectionSettings: true + AcceptsRemoteConfig: true + AcceptsRestartCommand: true + ReportsEffectiveConfig: true + ReportsHealth: true + ReportsOwnLogs: true + ReportsOwnMetrics: true + ReportsOwnTraces: true + ReportsRemoteConfig: true + ReportsStatus: true + componentsAllowed: + exporters: + - logging + processors: + - memory_limiter + receivers: + - otlp + endpoint: ws://e2e-test-app-bridge-server:4320/v1/opamp \ No newline at end of file diff --git a/tests/e2e-opampbridge/opampbridge/chainsaw-test.yaml b/tests/e2e-opampbridge/opampbridge/chainsaw-test.yaml index a428ab84db..b5c44bf376 100755 --- a/tests/e2e-opampbridge/opampbridge/chainsaw-test.yaml +++ b/tests/e2e-opampbridge/opampbridge/chainsaw-test.yaml @@ -14,5 +14,13 @@ spec: file: 00-assert.yaml catch: - podLogs: - selector: app.kubernetes.io/component=opentelemetry-opamp-bridge - \ No newline at end of file + selector: app=e2e-test-app-bridge-server + - name: step-01 + try: + - apply: + file: 01-install.yaml + - assert: + file: 01-assert.yaml + catch: + - podLogs: + selector: app.kubernetes.io/component=opentelemetry-opamp-bridge \ No newline at end of file From 9919075e11cfc546f7e2c7888c88aac4f409bf90 Mon Sep 17 00:00:00 2001 From: Jacob Aronoff Date: Mon, 8 Jul 2024 14:07:20 -0400 Subject: [PATCH 2/2] use the new bridge e2e test server --- Makefile | 2 +- .../opampbridge/01-assert.yaml | 2 +- .../opampbridge/01-install.yaml | 18 ++- .../opampbridge/02-assert.yaml | 56 +++++++++ .../opampbridge/02-install.yaml | 26 ++++ .../opampbridge/chainsaw-test.yaml | 114 +++++++++++++++--- .../bridge-server/opampsrv/opampsrv.go | 2 +- 7 files changed, 196 insertions(+), 24 deletions(-) create mode 100644 tests/e2e-opampbridge/opampbridge/02-assert.yaml create mode 100644 tests/e2e-opampbridge/opampbridge/02-install.yaml diff --git a/Makefile b/Makefile index b8e4fd3656..a9ae07a5d3 100644 --- a/Makefile +++ b/Makefile @@ -398,7 +398,7 @@ KUSTOMIZE_VERSION ?= v5.0.3 CONTROLLER_TOOLS_VERSION ?= v0.14.0 GOLANGCI_LINT_VERSION ?= v1.57.2 KIND_VERSION ?= v0.20.0 -CHAINSAW_VERSION ?= v0.1.7 +CHAINSAW_VERSION ?= v0.2.5 .PHONY: install-tools install-tools: kustomize golangci-lint kind controller-gen envtest crdoc kind operator-sdk chainsaw diff --git a/tests/e2e-opampbridge/opampbridge/01-assert.yaml b/tests/e2e-opampbridge/opampbridge/01-assert.yaml index bc576139e3..6e75572cd5 100644 --- a/tests/e2e-opampbridge/opampbridge/01-assert.yaml +++ b/tests/e2e-opampbridge/opampbridge/01-assert.yaml @@ -48,7 +48,7 @@ data: - memory_limiter receivers: - otlp - endpoint: ws://opamp-server:4320/v1/opamp + endpoint: ws://e2e-test-app-bridge-server:4320/v1/opamp kind: ConfigMap metadata: name: test-opamp-bridge diff --git a/tests/e2e-opampbridge/opampbridge/01-install.yaml b/tests/e2e-opampbridge/opampbridge/01-install.yaml index 2d5892df85..35e9b1cc02 100644 --- a/tests/e2e-opampbridge/opampbridge/01-install.yaml +++ b/tests/e2e-opampbridge/opampbridge/01-install.yaml @@ -1,11 +1,11 @@ apiVersion: v1 -automountServiceAccountToken: true kind: ServiceAccount +automountServiceAccountToken: true metadata: name: opamp-bridge --- apiVersion: rbac.authorization.k8s.io/v1 -kind: Role +kind: ClusterRole metadata: name: opamp-bridge rules: @@ -23,11 +23,25 @@ rules: - list - get --- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: bridge-cluster-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: opamp-bridge +subjects: + - kind: ServiceAccount + name: opamp-bridge + namespace: ($namespace) +--- apiVersion: opentelemetry.io/v1alpha1 kind: OpAMPBridge metadata: name: test spec: + serviceAccount: "opamp-bridge" capabilities: AcceptsOpAMPConnectionSettings: true AcceptsOtherConnectionSettings: true diff --git a/tests/e2e-opampbridge/opampbridge/02-assert.yaml b/tests/e2e-opampbridge/opampbridge/02-assert.yaml new file mode 100644 index 0000000000..d00252dd28 --- /dev/null +++ b/tests/e2e-opampbridge/opampbridge/02-assert.yaml @@ -0,0 +1,56 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: simplest-collector + labels: + opentelemetry.io/opamp-reporting: "true" +status: + readyReplicas: 1 + +--- + +apiVersion: v1 +kind: Service +metadata: + name: simplest-collector-headless +spec: + ports: + - appProtocol: grpc + name: jaeger-grpc + port: 14250 + protocol: TCP + targetPort: 14250 + - appProtocol: grpc + name: otlp-grpc + port: 4317 + protocol: TCP + targetPort: 4317 + - appProtocol: http + name: otlp-http + port: 4318 + protocol: TCP + targetPort: 4318 + +--- + +apiVersion: v1 +kind: Service +metadata: + name: simplest-collector +spec: + ports: + - appProtocol: grpc + name: jaeger-grpc + port: 14250 + protocol: TCP + targetPort: 14250 + - appProtocol: grpc + name: otlp-grpc + port: 4317 + protocol: TCP + targetPort: 4317 + - appProtocol: http + name: otlp-http + port: 4318 + protocol: TCP + targetPort: 4318 diff --git a/tests/e2e-opampbridge/opampbridge/02-install.yaml b/tests/e2e-opampbridge/opampbridge/02-install.yaml new file mode 100644 index 0000000000..288c025c06 --- /dev/null +++ b/tests/e2e-opampbridge/opampbridge/02-install.yaml @@ -0,0 +1,26 @@ +apiVersion: opentelemetry.io/v1alpha1 +kind: OpenTelemetryCollector +metadata: + name: simplest + labels: + opentelemetry.io/opamp-reporting: "true" +spec: + config: | + receivers: + jaeger: + protocols: + grpc: + otlp: + protocols: + grpc: + http: + processors: + + exporters: + debug: + + service: + pipelines: + traces: + receivers: [jaeger,otlp] + exporters: [debug] \ No newline at end of file diff --git a/tests/e2e-opampbridge/opampbridge/chainsaw-test.yaml b/tests/e2e-opampbridge/opampbridge/chainsaw-test.yaml index b5c44bf376..2d8a14249f 100755 --- a/tests/e2e-opampbridge/opampbridge/chainsaw-test.yaml +++ b/tests/e2e-opampbridge/opampbridge/chainsaw-test.yaml @@ -1,26 +1,102 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json apiVersion: chainsaw.kyverno.io/v1alpha1 kind: Test metadata: - creationTimestamp: null name: opampbridge spec: steps: - - name: step-00 - try: - - apply: - file: 00-install.yaml - - assert: - file: 00-assert.yaml - catch: - - podLogs: - selector: app=e2e-test-app-bridge-server - - name: step-01 - try: - - apply: - file: 01-install.yaml + - catch: + - podLogs: + selector: app=e2e-test-app-bridge-server + name: step-00 + try: + - apply: + file: 00-install.yaml + - assert: + file: 00-assert.yaml + - catch: + - podLogs: + selector: app.kubernetes.io/component=opentelemetry-opamp-bridge + name: step-01 + try: + - apply: + file: 01-install.yaml + - assert: + file: 01-assert.yaml + - name: Check effective config is empty for a valid agent id + try: + - script: + content: | + #!/bin/bash + # set -ex + # bridge_server_host=$(kubectl get service e2e-test-app-bridge-server -n $NAMESPACE -o jsonpath='{.spec.clusterIP}') + # curl -H "Content-Type: application/json" http://${bridge_server_host}:4321/agents + # TODO: Uncomment the above when proxying is available in chainsaw + kubectl get --raw /api/v1/namespaces/$NAMESPACE/services/e2e-test-app-bridge-server:4321/proxy/agents + outputs: + - name: result + value: (json_parse($stdout)) + - assert: + resource: + (length(values($result))): 1 + (values($result)[0].status.sequence_num >= `2`): true + (values($result)[0].status.health.healthy): true + (length(values($result)[0].effective_config)): 0 + - catch: + - podLogs: + selector: app.kubernetes.io/component=opentelemetry-opamp-bridge + name: step-02 + try: + - apply: + file: 02-install.yaml + - assert: + file: 02-assert.yaml + - try: + - sleep: + duration: 1s + # Right now the heartbeat interval isn't configurable by the CRD for the bridge + # This means that it can take 45s to inform the server of the new collector + # By deleting the pod we can just shortcut that + - try: + - script: + content: | + #!/bin/bash + kubectl delete pod -l app.kubernetes.io/name=test-opamp-bridge -n $NAMESPACE + - try: + - sleep: + duration: 3s + - try: - assert: - file: 01-assert.yaml - catch: - - podLogs: - selector: app.kubernetes.io/component=opentelemetry-opamp-bridge \ No newline at end of file + # specify resource inline + resource: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: test-opamp-bridge + status: + readyReplicas: 1 + - name: Check effective config is not empty + try: + - script: + content: | + #!/bin/bash + # set -ex + # bridge_server_host=$(kubectl get service e2e-test-app-bridge-server -n $NAMESPACE -o jsonpath='{.spec.clusterIP}') + # curl -H "Content-Type: application/json" http://${bridge_server_host}:4321/agents + # TODO: Uncomment the above when proxying is available in chainsaw + kubectl get --raw /api/v1/namespaces/$NAMESPACE/services/e2e-test-app-bridge-server:4321/proxy/agents + outputs: + - name: result + value: (json_parse($stdout)) + - assert: + resource: + # Note: often times here I need to assert the length and then get the 0th element + # it would be great if I could wildcard for the initial instance id key and then + # construct a compound key of `$NAMESPACE/simplest` for the collector I want to assert + (length(values($result))): 1 + (values($result)[0].status.sequence_num >= `2`): true + (values($result)[0].status.health.healthy): true + (length(values($result)[0].status.health.component_health_map)): 1 + (values(values($result)[0].status.health.component_health_map)[0].healthy): true + (length(values($result)[0].effective_config)): 1 + (length(values($result)[0].status.effective_config.config_map.config_map)): 1 + (length(values(values($result)[0].status.effective_config.config_map.config_map)[0].body) > `0`): true diff --git a/tests/test-e2e-apps/bridge-server/opampsrv/opampsrv.go b/tests/test-e2e-apps/bridge-server/opampsrv/opampsrv.go index 99f4c008c3..380b29e8e0 100644 --- a/tests/test-e2e-apps/bridge-server/opampsrv/opampsrv.go +++ b/tests/test-e2e-apps/bridge-server/opampsrv/opampsrv.go @@ -60,7 +60,7 @@ func (srv *Server) Start() { }, }, }, - ListenEndpoint: "127.0.0.1:4320", + ListenEndpoint: "0.0.0.0:4320", HTTPMiddleware: otelhttp.NewMiddleware("/v1/opamp"), }