Export logs to in-cluster logging (open-telemetry#3217)

Author: IshwarKanse

tests/e2e-openshift/Dockerfile

@@ -18,7 +18,7 @@ RUN mkdir -p /tmp/go/bin $GOCACHE \
     && chmod -R 777 /tmp/go/bin $GOPATH $GOCACHE
 
 # Install dependencies required by test cases and debugging
-RUN apt-get update && apt-get install -y jq vim libreadline-dev
+RUN apt-get update && apt-get install -y jq vim libreadline-dev unzip
 
 # Install kuttl e2e
 RUN curl -LO https://github.com/kudobuilder/kuttl/releases/download/v0.15.0/kubectl-kuttl_0.15.0_linux_x86_64 \
@@ -34,5 +34,11 @@ RUN curl -LO https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp/la
     && chmod +x oc kubectl \
     && mv oc kubectl /usr/local/bin/
 
+# Install the latest version of logcli
+RUN curl -LO https://github.com/grafana/loki/releases/latest/download/logcli-linux-amd64.zip \
+    && unzip logcli-linux-amd64.zip \
+    && chmod +x logcli-linux-amd64 \
+    && mv logcli-linux-amd64 /usr/local/bin/logcli
+
 # Set the working directory
 WORKDIR /tmp/opentelemetry-operator

tests/e2e-openshift/export-to-cluster-logging-lokistack/chainsaw-test.yaml

@@ -0,0 +1,62 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  name: export-to-cluster-logging-lokistack
+spec:
+  namespace: chainsaw-incllogs
+  description: Tests shipping of logs to OpenShift in-cluster Logging LokiStack instance using OpenTelemetry collector. The tests requires the Loki, OpenTelemetry and Cluster Observability Operator to be installed along with logcli
+  steps:
+  - name: Create the OTEL collector instance
+    try:
+    - apply:
+        file: otel-collector.yaml
+    - assert:
+        file: otel-collector-assert.yaml
+  - name: Install Minio instance
+    try:
+    - apply:
+        file: install-minio.yaml
+    - assert:
+        file: install-minio-assert.yaml
+  - name: Create the LokiStack instance
+    try:
+    - apply:
+        file: install-loki.yaml
+    - assert:
+        file: install-loki-assert.yaml
+  - name: Check the status of LokiStack instance
+    try:
+    - script:
+        timeout: 5m
+        content: kubectl get --namespace openshift-logging lokistacks logging-loki -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}'
+  - name: Enable Logging UI. Not needed by test but can be used for debugging
+    try:
+    - apply:
+        file: logging-uiplugin.yaml
+    - assert:
+        file: logging-uiplugin-assert.yaml
+  - name: Generate logs
+    try:
+    - apply:
+        file: generate-logs.yaml
+    - assert:
+        file: generate-logs-assert.yaml
+  - name: Check logs in LokiStack instance
+    try:
+    - script:
+        timeout: 5m
+        content: ./check_logs.sh
+    cleanup:
+    - delete:
+        ref:
+          apiVersion: loki.grafana.com/v1
+          kind: LokiStack
+          name: logging-loki
+          namespace: openshift-logging
+    - delete:
+        ref:
+          apiVersion: v1
+          kind: PersistentVolumeClaim
+          labels: 
+            app.kubernetes.io/instance: "logging-loki"
+          namespace: openshift-logging

tests/e2e-openshift/export-to-cluster-logging-lokistack/check_logs.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+TOKEN=$(oc -n openshift-logging create token otel-collector-deployment)
+LOKI_URL=$(oc -n openshift-logging get route logging-loki -o json | jq '.spec.host' -r)
+
+while true; do
+  LOG_OUTPUT=$(logcli -o raw --tls-skip-verify \
+  --bearer-token="${TOKEN}" \
+  --addr "https://${LOKI_URL}/api/logs/v1/application" query '{log_type="application"}')
+
+  if echo "$LOG_OUTPUT" | jq -e '
+    . as $root |
+    select(
+      .body == "the message" and
+      .severity == "Info" and
+      .attributes.app == "server" and
+      .resources."k8s.container.name" == "telemetrygen" and
+      .resources."k8s.namespace.name" == "chainsaw-incllogs"
+    )
+  ' > /dev/null; then
+    echo "Logs found:"
+    break
+  else
+    echo "Logs not found. Continuing to check..."
+    sleep 5
+  fi
+done

tests/e2e-openshift/export-to-cluster-logging-lokistack/generate-logs-assert.yaml

@@ -0,0 +1,7 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: telemetrygen
+  namespace: chainsaw-incllogs
+status:
+  active: 1

tests/e2e-openshift/export-to-cluster-logging-lokistack/generate-logs.yaml

@@ -0,0 +1,20 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: telemetrygen
+  namespace: chainsaw-incllogs
+spec:
+  template:
+    spec:
+      containers:
+        - name: telemetrygen
+          image: ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:v0.106.1
+          args:
+            - logs
+            - --otlp-endpoint=otel-collector.openshift-logging.svc.cluster.local:4317
+            - --otlp-insecure
+            - --workers=1
+            - --duration=120s
+            - --otlp-attributes=k8s.container.name="telemetrygen"
+      restartPolicy: Never
+  backoffLimit: 4