Convert test back to a job and break out functionality for the config analysis (#397)

Signed-off-by: Pete Wall <[email protected]>

Author: petewall

.github/configs/mysql-config.yaml

@@ -60,6 +60,9 @@ data:
     prometheus.scrape "mysql" {
       targets      = prometheus.exporter.mysql.mysql.targets
       job_name     = "integrations/mysql"
+      clustering {
+        enabled = true
+      }
       forward_to   = [prometheus.relabel.mysql.receiver]
     }
 

.github/workflows/helm-test.yml

@@ -201,4 +201,4 @@ jobs:
       if: steps.list-changed.outputs.changed == 'true'
       run: |
         latestRelease=$(git describe --abbrev=0 --tags)
-        ct install --config "${CT_CONFIGFILE}" --since "${latestRelease}"
+        ct install --config "${CT_CONFIGFILE}" --since "${latestRelease}" --helm-extra-args "--timeout 10m"

charts/k8s-monitoring/README.md

@@ -111,6 +111,16 @@ The Prometheus and Loki services may be hosted on the same cluster, or remotely
 | cluster.kubernetesAPIService | string | `"kubernetes.default.svc.cluster.local:443"` | The Kubernetes service. Change this if your cluster DNS is configured differently than the default. |
 | cluster.name | string | `""` | The name of this cluster, which will be set in all labels. Required. |
 | cluster.platform | string | `""` | The specific platform for this cluster. Will enable compatibility for some platforms. Supported options: (empty) or "openshift". |
+| configAnalysis.enabled | bool | `true` | Should `helm test` run the config analysis pod? |
+| configAnalysis.extraAnnotations | object | `{}` | Extra annotations to add to the config analysis pod. |
+| configAnalysis.extraLabels | object | `{}` | Extra labels to add to the config analysis pod. |
+| configAnalysis.image.image | string | `"grafana/k8s-monitoring-test"` | Config Analysis image repository. |
+| configAnalysis.image.pullSecrets | list | `[]` | Optional set of image pull secrets. |
+| configAnalysis.image.registry | string | `"ghcr.io"` | Config Analysis image registry. |
+| configAnalysis.image.tag | string | `""` | Config Analysis image tag. Default is the chart version. |
+| configAnalysis.nodeSelector | object | `{"kubernetes.io/os":"linux"}` | nodeSelector to apply to the config analysis pod. |
+| configAnalysis.tolerations | list | `[]` | Tolerations to apply to the config analysis pod. |
+| configValidator.enabled | bool | `true` | Should config validation be run? |
 | configValidator.extraAnnotations | object | `{}` | Extra annotations to add to the test config validator job. |
 | configValidator.extraLabels | object | `{}` | Extra labels to add to the test config validator job. |
 | configValidator.nodeSelector | object | `{"kubernetes.io/os":"linux"}` | nodeSelector to apply to the config validator job. |
@@ -354,10 +364,11 @@ The Prometheus and Loki services may be hosted on the same cluster, or remotely
 | receivers.zipkin.enabled | bool | `false` | Receive Zipkin traces |
 | receivers.zipkin.port | int | `9411` | Which port to use for the Zipkin receiver. This port needs to be opened in the grafana-agent section below. |
 | test.attempts | int | `10` | How many times to attempt the test job. |
+| test.enabled | bool | `true` | Should `helm test` run the test job? |
 | test.envOverrides | object | `{"LOKI_URL":"","PROMETHEUS_URL":"","TEMPO_URL":""}` | Overrides the URLs for various data sources |
-| test.extraAnnotations | object | `{}` | Extra annotations to add to the test jobs. |
-| test.extraLabels | object | `{}` | Extra labels to add to the test jobs. |
-| test.extraQueries | list | `[]` | Additional queries that will be run with `helm test`. NOTE that this uses the host, username, and password in the externalServices section. The user account must have the ability to run queries. Example: extraQueries:   - query: prometheus_metric{cluster="my-cluster-name"}     type: [promql|logql] |
+| test.extraAnnotations | object | `{}` | Extra annotations to add to the test job. |
+| test.extraLabels | object | `{}` | Extra labels to add to the test job. |
+| test.extraQueries | list | `[]` | Additional queries to run during the test. NOTE that this uses the host, username, and password in the externalServices section. The user account must have the ability to run queries. Example: extraQueries:   - query: prometheus_metric{cluster="my-cluster-name"}     type: promql  Can optionally provide expectations: - query: "avg(count_over_time(scrape_samples_scraped{cluster=~\"ci-test-cluster-2|from-the-other-agent\"}[1m]))"   type: promql   expect:     value: 1     operator: == |
 | test.image.image | string | `"grafana/k8s-monitoring-test"` | Test job image repository. |
 | test.image.pullSecrets | list | `[]` | Optional set of image pull secrets. |
 | test.image.registry | string | `"ghcr.io"` | Test job image registry. |

charts/k8s-monitoring/ci/ci-2-values.yaml

@@ -80,11 +80,14 @@ test:
       operator: "<="  # There's gotta be at least one pod
 
   # Check for cluster events
-  - query: "{cluster=\"ci-test-cluster-2\", job=\"integrations/kubernetes/eventhandler\"}"
+  - query: "count_over_time({cluster=\"ci-test-cluster-2\", job=\"integrations/kubernetes/eventhandler\"}[1h])"
     type: logql
+
   # Check for pod logs (gathered via API)
-  - query: "{cluster=\"ci-test-cluster-2\", job!=\"integrations/kubernetes/eventhandler\"}"
+  - query: "count_over_time({cluster=\"ci-test-cluster-2\", job!=\"integrations/kubernetes/eventhandler\"}[1h])"
     type: logql
+
+  # Check for traces
   - query: "{.k8s.cluster.name=\"ci-test-cluster-2\"}"
     type: traceql
 

charts/k8s-monitoring/ci/ci-integrations-values.yaml

@@ -106,15 +106,15 @@ test:
     type: promql
 
   # Check for MySQL logs, discovered by the module loaded above in .logs.extraConfig
-  - query: "{cluster=\"ci-integrations-cluster\", job=\"integrations/mysql\"}"
+  - query: "count_over_time({cluster=\"ci-integrations-cluster\", job=\"integrations/mysql\"}[1h])"
     type: logql
 
   # Cluster events still work, too
-  - query: "{cluster=\"ci-integrations-cluster\", job=\"integrations/kubernetes/eventhandler\"}"
+  - query: "count_over_time({cluster=\"ci-integrations-cluster\", job=\"integrations/kubernetes/eventhandler\"}[1h])"
     type: logql
 
   # Regular pod logs still work, too (checking the Loki pod <namespace>/<pod_name>)
-  - query: "{cluster=\"ci-integrations-cluster\", job=\"loki/loki\"}"
+  - query: "count_over_time({cluster=\"ci-integrations-cluster\", job=\"loki/loki\"}[1h])"
     type: logql
 
   # DPM check

charts/k8s-monitoring/ci/ci-values.yaml

@@ -63,10 +63,10 @@ test:
       expect:
         count: 0
     # Check for cluster events
-    - query: "{cluster=\"ci-test-cluster\", job=\"integrations/kubernetes/eventhandler\"}"
+    - query: "count_over_time({cluster=\"ci-test-cluster\", job=\"integrations/kubernetes/eventhandler\"}[1h])"
       type: logql
     # Check for pod logs
-    - query: "{cluster=\"ci-test-cluster\", job!=\"integrations/kubernetes/eventhandler\"}"
+    - query: "count_over_time({cluster=\"ci-test-cluster\", job!=\"integrations/kubernetes/eventhandler\"}[1h])"
       type: logql
 
     # DPM check

charts/k8s-monitoring/templates/hooks/validate-configuration.yaml

@@ -1,3 +1,4 @@
+{{- if .Values.configValidator.enabled }}
 ---
 apiVersion: batch/v1
 kind: Job
@@ -112,3 +113,4 @@ data:
   logs.river: |-
     {{- include "agentLogsConfig" . | trim | nindent 4 }}
 {{- end }}
+{{- end -}}