Skip to content

Commit 45a95dc

Browse files
authored
Add eks addon tests (#1304)
* Add eks addon tests Signed-off-by: Pete Wall <[email protected]> * Don't run addon tests automatically, since they don't actually test the current version of the chart Signed-off-by: Pete Wall <[email protected]> * Add the ability to skip running the integration tests and skip rendering on certain platform tests Signed-off-by: Pete Wall <[email protected]> * Install the addon with the AWS cli, rather than eksctl. More closely mimics what a customer might do Signed-off-by: Pete Wall <[email protected]> * Remove unused gitignore Signed-off-by: Pete Wall <[email protected]> --------- Signed-off-by: Pete Wall <[email protected]>
1 parent 0deb8eb commit 45a95dc

File tree

26 files changed

+371
-3
lines changed

26 files changed

+371
-3
lines changed

.github/workflows/integration-test.yml

+9
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,15 @@ jobs:
2929
id: list_tests
3030
working-directory: charts/k8s-monitoring/tests/integration
3131
run: |
32+
# if "integration-test-skip" is set, return an empty list
33+
# All labels on this PR
34+
labels='${{ toJson(github.event.pull_request.labels.*.name) }}'
35+
if echo "${labels}" | jq --raw-output --compact-output 'index("integration-test-skip")' > /dev/null; then
36+
echo "\"integration-test-skip\" label is set, skipping integration tests."
37+
echo "tests=[]" >> "${GITHUB_OUTPUT}"
38+
exit 0
39+
fi
40+
3241
tests=$(find . -name values.yaml -exec dirname {} \;)
3342
echo "Tests: ${tests}"
3443
echo "tests=$(echo "${tests}" | jq --raw-input --slurp --compact-output 'split("\n") | map(select(. != ""))')" >> "${GITHUB_OUTPUT}"

.github/workflows/platform-test.yml

+3-1
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ jobs:
2525
working-directory: charts/k8s-monitoring/tests/platform
2626
run: |
2727
allTests=$(find . -name values.yaml -type f -exec dirname {} \; | sed "s|^\./||" | jq --raw-input --slurp --compact-output 'split("\n") | map(select(. != ""))')
28+
manualOnlyTests=$(find . -name .manual-only -type f -exec dirname {} \; | sed "s|^\./||" | jq --raw-input --slurp --compact-output 'split("\n") | map(select(. != ""))')
2829
if [ "${{ github.event_name }}" == "pull_request" ]; then
2930
# All labels on this PR
3031
labels='${{ toJson(github.event.pull_request.labels.*.name) }}'
@@ -35,7 +36,8 @@ jobs:
3536
# Choose the tests that match the labels
3637
tests=$(jq --null-input --compact-output --argjson allTests "${allTests}" --argjson chosenTests "${chosenTests}" '$allTests | map(select(. as $test | $chosenTests | index($test)))')
3738
else
38-
tests="${allTests}"
39+
# remove manual only tests
40+
tests=$(jq --null-input --compact-output --argjson allTests "${allTests}" --argjson manualOnlyTests "${manualOnlyTests}" '$allTests | map(select(. as $test | $manualOnlyTests | index($test) | not))')
3941
fi
4042
echo "Running tests: ${tests}"
4143
echo "tests=${tests}" >> "${GITHUB_OUTPUT}"

charts/k8s-monitoring/Makefile

+3-2
Original file line numberDiff line numberDiff line change
@@ -90,11 +90,12 @@ EXAMPLE_OUTPUT_FILES = $(EXAMPLE_VALUES_FILES:values.yaml=output.yaml)
9090
EXAMPLE_ALLOY_FILES = $(foreach file,$(EXAMPLE_VALUES_FILES),$(call alloy_configs, $(file)))
9191
EXAMPLE_README_FILES = $(EXAMPLE_VALUES_FILES:values.yaml=README.md)
9292

93-
NON_RENDERED_INTEGRATION_TEST_VALUES_FILES = $(shell find tests/integration -name .norender | sed 's/.norender/values.yaml/')
93+
NON_RENDERED_INTEGRATION_TEST_VALUES_FILES = $(shell find tests/integration -name .no-render | sed 's/.no-render/values.yaml/')
9494
INTEGRATION_TEST_VALUES_FILES = $(filter-out $(NON_RENDERED_INTEGRATION_TEST_VALUES_FILES),$(shell find tests/integration -name values.yaml))
9595
INTEGRATION_TEST_OUTPUT_FILES = $(INTEGRATION_TEST_VALUES_FILES:values.yaml=.rendered/output.yaml)
9696

97-
PLATFORM_TEST_VALUES_FILES = $(shell find tests/platform -name values.yaml)
97+
NON_RENDERED_PLATFORM_TEST_VALUES_FILES = $(shell find tests/platform -name .no-render | sed 's/.no-render/values.yaml/')
98+
PLATFORM_TEST_VALUES_FILES = $(filter-out $(NON_RENDERED_PLATFORM_TEST_VALUES_FILES),$(shell find tests/platform -name values.yaml))
9899
PLATFORM_TEST_OUTPUT_FILES = $(PLATFORM_TEST_VALUES_FILES:values.yaml=.rendered/output.yaml)
99100

100101
alloy_configs = $(shell \
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
AWS_ACCESS_KEY_ID=$(op --account grafana.1password.com read "op://Kubernetes Monitoring/AWS Access Key/access key id")
2+
AWS_SECRET_ACCESS_KEY=$(op --account grafana.1password.com read "op://Kubernetes Monitoring/AWS Access Key/secret access key")
3+
export AWS_ACCESS_KEY_ID
4+
export AWS_SECRET_ACCESS_KEY
5+
export AWS_DEFAULT_REGION=ap-northeast-2
6+
export KUBECONFIG=$(pwd)/kubeconfig.yaml
7+
8+
export GRAFANA_CLOUD_METRICS_USERNAME=$(op --account grafana.1password.com read "op://Kubernetes Monitoring/helmchart Prometheus/username")
9+
export GRAFANA_CLOUD_LOGS_USERNAME=$(op --account grafana.1password.com read "op://Kubernetes Monitoring/helmchart Loki/username")
10+
export GRAFANA_CLOUD_TRACES_USERNAME=$(op --account grafana.1password.com read "op://Kubernetes Monitoring/helmchart Tempo/username")
11+
export GRAFANA_CLOUD_RW_POLICY_TOKEN=$(op --account grafana.1password.com read "op://Kubernetes Monitoring/helmchart Prometheus/password")
12+
export RANDOM_NUMBER=$(shuf -i 100000-999999 -n 1)
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
.terraform
2+
.terraform.lock.hcl
3+
kubeconfig.yaml
4+
terraform.tfstate*
5+
vars.tf
6+
deps.json
7+
deployments/grafana-cloud.yaml
8+
deployments/grafana-cloud-credentials.yaml
9+
deployments/test-variables.yaml

charts/k8s-monitoring/tests/platform/eks-addon-with-terraform/.manual-only

Whitespace-only changes.

charts/k8s-monitoring/tests/platform/eks-addon-with-terraform/.no-render

Whitespace-only changes.
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
all: deployments/test-variables.yaml deployments/grafana-cloud.yaml deployments/grafana-cloud-credentials.yaml vars.tf
2+
clean:
3+
rm -f deployments/test-variables.yaml deployments/grafana-cloud.yaml deployments/grafana-cloud-credentials.yaml vars.tf
4+
5+
deployments/test-variables.yaml:
6+
echo "---" > $@
7+
kubectl create configmap test-variables \
8+
--from-literal=CLUSTER="$(shell yq eval '.cluster.name' values.yaml)-$$RANDOM_NUMBER" \
9+
--from-literal=RANDOM_NUMBER="$$RANDOM_NUMBER" \
10+
-o yaml --dry-run=client >> $@
11+
12+
deployments/grafana-cloud.yaml:
13+
echo "---" > $@
14+
kubectl create namespace monitoring --dry-run=client -o yaml >> $@
15+
echo "---" >> $@
16+
echo "# yamllint disable rule:line-length" >> $@
17+
kubectl create secret generic grafana-cloud \
18+
--namespace=monitoring \
19+
--from-literal=prometheus-host="https://prometheus-prod-13-prod-us-east-0.grafana.net" \
20+
--from-literal=prometheus-username="$$GRAFANA_CLOUD_METRICS_USERNAME" \
21+
--from-literal=prometheus-password="$$GRAFANA_CLOUD_RW_POLICY_TOKEN" \
22+
--from-literal=loki-host="https://logs-prod-006.grafana.net" \
23+
--from-literal=loki-username="$$GRAFANA_CLOUD_LOGS_USERNAME" \
24+
--from-literal=loki-password="$$GRAFANA_CLOUD_RW_POLICY_TOKEN" \
25+
--from-literal=tempo-host="https://tempo-prod-04-prod-us-east-0.grafana.net:443" \
26+
--from-literal=tempo-username="$$GRAFANA_CLOUD_TRACES_USERNAME" \
27+
--from-literal=tempo-password="$$GRAFANA_CLOUD_RW_POLICY_TOKEN" \
28+
-o yaml --dry-run=client >> $@
29+
30+
deployments/grafana-cloud-credentials.yaml:
31+
echo "---" > $@
32+
echo "# yamllint disable rule:line-length" >> $@
33+
kubectl create secret generic grafana-cloud-credentials \
34+
--from-literal=PROMETHEUS_USER="$$GRAFANA_CLOUD_METRICS_USERNAME" \
35+
--from-literal=PROMETHEUS_PASS="$$GRAFANA_CLOUD_RW_POLICY_TOKEN" \
36+
--from-literal=LOKI_USER="$$GRAFANA_CLOUD_LOGS_USERNAME" \
37+
--from-literal=LOKI_PASS="$$GRAFANA_CLOUD_RW_POLICY_TOKEN" \
38+
-o yaml --dry-run=client >> $@
39+
40+
vars.tf:
41+
echo "variable \"cluster-name\" {" > $@
42+
echo " type = string" >> $@
43+
echo " default = \"$(shell yq eval '.cluster.name' values.yaml)-$$RANDOM_NUMBER\"" >> $@
44+
echo "}" >> $@
45+
echo "" >> $@
46+
47+
echo "variable \"aws-access-key\" {" >> $@
48+
echo " type = string" >> $@
49+
echo " default = \"$$AWS_ACCESS_KEY_ID\"" >> $@
50+
echo " sensitive = true" >> $@
51+
echo "}" >> $@
52+
echo "" >> $@
53+
54+
echo "variable \"aws-secret-key\" {" >> $@
55+
echo " type = string" >> $@
56+
echo " default = \"$$AWS_SECRET_ACCESS_KEY\"" >> $@
57+
echo " sensitive = true" >> $@
58+
echo "}" >> $@
59+
60+
61+
run-test:
62+
../../../../../scripts/run-cluster-test.sh .
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
resource "aws_eks_addon" "default" {
2+
cluster_name = var.cluster-name
3+
addon_name = "grafana-labs_kubernetes-monitoring"
4+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
#!/usr/bin/env bash
2+
3+
TEST_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
4+
pushd "${TEST_DIR}" || exit 1
5+
6+
terraform init
7+
terraform apply -auto-approve
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
---
2+
apiVersion: source.toolkit.fluxcd.io/v1
3+
kind: GitRepository
4+
metadata:
5+
name: k8s-monitoring-test
6+
spec:
7+
interval: 1m
8+
url: https://github.com/grafana/k8s-monitoring-helm
9+
ref:
10+
branch: main
11+
ignore: |
12+
/*
13+
!/charts/k8s-monitoring-test
14+
---
15+
apiVersion: helm.toolkit.fluxcd.io/v2
16+
kind: HelmRelease
17+
metadata:
18+
name: k8s-monitoring-test
19+
spec:
20+
interval: 1m
21+
chart:
22+
spec:
23+
chart: charts/k8s-monitoring-test
24+
sourceRef:
25+
kind: GitRepository
26+
name: k8s-monitoring-test
27+
interval: 1m
28+
values:
29+
tests:
30+
- env:
31+
PROMETHEUS_URL: https://prometheus-prod-13-prod-us-east-0.grafana.net/api/prom/api/v1/query
32+
LOKI_URL: https://logs-prod-006.grafana.net/loki/api/v1/query
33+
envFrom:
34+
- secretRef: {name: grafana-cloud-credentials}
35+
- configMapRef: {name: test-variables}
36+
queries:
37+
# Cluster metrics
38+
- query: kubernetes_build_info{cluster="$CLUSTER", job="integrations/kubernetes/kubelet"}
39+
type: promql
40+
- query: node_cpu_usage_seconds_total{cluster="$CLUSTER", job="integrations/kubernetes/resources"}
41+
type: promql
42+
- query: machine_memory_bytes{cluster="$CLUSTER", job="integrations/kubernetes/cadvisor"}
43+
type: promql
44+
- query: count(kube_node_info{cluster="$CLUSTER", job="integrations/kubernetes/kube-state-metrics"})
45+
type: promql
46+
47+
# Cluster events
48+
- query: count_over_time({cluster="$CLUSTER", job="integrations/kubernetes/eventhandler"}[1h])
49+
type: logql
50+
51+
# Pod logs
52+
- query: count_over_time({cluster="$CLUSTER", job!~"integrations/kubernetes/eventhandler|integrations/kubernetes/journal"}[1h])
53+
type: logql
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
---
2+
kind: ClusterConfig
3+
apiVersion: eksctl.io/v1alpha5
4+
metadata:
5+
region: ap-northeast-2
6+
tags:
7+
source: k8s-monitoring-helm-platform-test
8+
iam:
9+
withOIDC: true
10+
nodeGroups:
11+
- name: ng-linux
12+
instanceType: m5.large
13+
minSize: 1
14+
maxSize: 1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
terraform {
2+
required_providers {
3+
aws = {
4+
source = "hashicorp/aws"
5+
version = "5.90.0"
6+
}
7+
}
8+
}
9+
10+
provider "aws" {
11+
access_key = var.aws-access-key
12+
region = "ap-northeast-2"
13+
secret_key = var.aws-secret-key
14+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
---
2+
cluster:
3+
name: eks-addon-with-terraform
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
AWS_ACCESS_KEY_ID=$(op --account grafana.1password.com read "op://Kubernetes Monitoring/AWS Access Key/access key id")
2+
AWS_SECRET_ACCESS_KEY=$(op --account grafana.1password.com read "op://Kubernetes Monitoring/AWS Access Key/secret access key")
3+
export AWS_ACCESS_KEY_ID
4+
export AWS_SECRET_ACCESS_KEY
5+
export AWS_DEFAULT_REGION=ap-northeast-2
6+
export KUBECONFIG=$(pwd)/kubeconfig.yaml
7+
8+
export GRAFANA_CLOUD_METRICS_USERNAME=$(op --account grafana.1password.com read "op://Kubernetes Monitoring/helmchart Prometheus/username")
9+
export GRAFANA_CLOUD_LOGS_USERNAME=$(op --account grafana.1password.com read "op://Kubernetes Monitoring/helmchart Loki/username")
10+
export GRAFANA_CLOUD_TRACES_USERNAME=$(op --account grafana.1password.com read "op://Kubernetes Monitoring/helmchart Tempo/username")
11+
export GRAFANA_CLOUD_RW_POLICY_TOKEN=$(op --account grafana.1password.com read "op://Kubernetes Monitoring/helmchart Prometheus/password")
12+
export RANDOM_NUMBER=$(shuf -i 100000-999999 -n 1)
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
kubeconfig.yaml
2+
deployments/grafana-cloud.yaml
3+
deployments/grafana-cloud-credentials.yaml
4+
deployments/test-variables.yaml

charts/k8s-monitoring/tests/platform/eks-addon/.manual-only

Whitespace-only changes.

charts/k8s-monitoring/tests/platform/eks-addon/.no-render

Whitespace-only changes.

charts/k8s-monitoring/tests/platform/eks-addon/.rendered/output.yaml

Whitespace-only changes.
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
all: values.json deployments/test-variables.yaml deployments/grafana-cloud.yaml deployments/grafana-cloud-credentials.yaml
2+
clean:
3+
rm -f deployments/test-variables.yaml deployments/grafana-cloud.yaml deployments/grafana-cloud-credentials.yaml
4+
5+
values.json: values.yaml
6+
yq eval --output-format json '. | del(.cluster.name)' values.yaml > $@
7+
8+
deployments/test-variables.yaml:
9+
echo "---" > $@
10+
kubectl create configmap test-variables \
11+
--from-literal=CLUSTER="$(shell yq eval '.cluster.name' values.yaml)-$$RANDOM_NUMBER" \
12+
--from-literal=RANDOM_NUMBER="$$RANDOM_NUMBER" \
13+
-o yaml --dry-run=client >> $@
14+
15+
deployments/grafana-cloud.yaml:
16+
echo "---" > $@
17+
kubectl create namespace monitoring --dry-run=client -o yaml >> $@
18+
echo "---" >> $@
19+
echo "# yamllint disable rule:line-length" >> $@
20+
kubectl create secret generic grafana-cloud \
21+
--namespace=monitoring \
22+
--from-literal=prometheus-host="https://prometheus-prod-13-prod-us-east-0.grafana.net" \
23+
--from-literal=prometheus-username="$$GRAFANA_CLOUD_METRICS_USERNAME" \
24+
--from-literal=prometheus-password="$$GRAFANA_CLOUD_RW_POLICY_TOKEN" \
25+
--from-literal=loki-host="https://logs-prod-006.grafana.net" \
26+
--from-literal=loki-username="$$GRAFANA_CLOUD_LOGS_USERNAME" \
27+
--from-literal=loki-password="$$GRAFANA_CLOUD_RW_POLICY_TOKEN" \
28+
--from-literal=tempo-host="https://tempo-prod-04-prod-us-east-0.grafana.net:443" \
29+
--from-literal=tempo-username="$$GRAFANA_CLOUD_TRACES_USERNAME" \
30+
--from-literal=tempo-password="$$GRAFANA_CLOUD_RW_POLICY_TOKEN" \
31+
-o yaml --dry-run=client >> $@
32+
33+
deployments/grafana-cloud-credentials.yaml:
34+
echo "---" > $@
35+
echo "# yamllint disable rule:line-length" >> $@
36+
kubectl create secret generic grafana-cloud-credentials \
37+
--from-literal=PROMETHEUS_USER="$$GRAFANA_CLOUD_METRICS_USERNAME" \
38+
--from-literal=PROMETHEUS_PASS="$$GRAFANA_CLOUD_RW_POLICY_TOKEN" \
39+
--from-literal=LOKI_USER="$$GRAFANA_CLOUD_LOGS_USERNAME" \
40+
--from-literal=LOKI_PASS="$$GRAFANA_CLOUD_RW_POLICY_TOKEN" \
41+
-o yaml --dry-run=client >> $@
42+
43+
run-test:
44+
../../../../../scripts/run-cluster-test.sh .
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
#!/usr/bin/env bash
2+
3+
TEST_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
4+
pushd "${TEST_DIR}" || exit 1
5+
6+
CLUSTER_NAME="$(yq eval '.cluster.name' "values.yaml")-${RANDOM_NUMBER}"
7+
CLUSTER_REGION=$(yq eval '.metadata.region' eks-cluster-config.yaml)
8+
ADDON_NAME=grafana-labs_kubernetes-monitoring
9+
10+
# Check if the addon already exists
11+
if aws eks list-addons --cluster-name "${CLUSTER_NAME}" --region "${CLUSTER_REGION}" | jq -e ".addons | index(\"${ADDON_NAME}\")" > /dev/null; then
12+
echo "Updating addon, \"${ADDON_NAME}\" in cluster ${CLUSTER_NAME}..."
13+
aws eks update-addon \
14+
--cluster-name "${CLUSTER_NAME}" \
15+
--region "${CLUSTER_REGION}" \
16+
--addon-name "${ADDON_NAME}" \
17+
--configuration-values "$(jq --compact-output . values.json)"
18+
else
19+
echo "installing addon, \"${ADDON_NAME}\" in cluster ${CLUSTER_NAME}..."
20+
aws eks create-addon \
21+
--cluster-name "${CLUSTER_NAME}" \
22+
--region "${CLUSTER_REGION}" \
23+
--addon-name "${ADDON_NAME}" \
24+
--configuration-values "$(jq --compact-output . values.json)"
25+
fi
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
---
2+
apiVersion: source.toolkit.fluxcd.io/v1
3+
kind: GitRepository
4+
metadata:
5+
name: k8s-monitoring-test
6+
spec:
7+
interval: 1m
8+
url: https://github.com/grafana/k8s-monitoring-helm
9+
ref:
10+
branch: main
11+
ignore: |
12+
/*
13+
!/charts/k8s-monitoring-test
14+
---
15+
apiVersion: helm.toolkit.fluxcd.io/v2
16+
kind: HelmRelease
17+
metadata:
18+
name: k8s-monitoring-test
19+
spec:
20+
interval: 1m
21+
chart:
22+
spec:
23+
chart: charts/k8s-monitoring-test
24+
sourceRef:
25+
kind: GitRepository
26+
name: k8s-monitoring-test
27+
interval: 1m
28+
values:
29+
tests:
30+
- env:
31+
PROMETHEUS_URL: https://prometheus-prod-13-prod-us-east-0.grafana.net/api/prom/api/v1/query
32+
LOKI_URL: https://logs-prod-006.grafana.net/loki/api/v1/query
33+
envFrom:
34+
- secretRef: {name: grafana-cloud-credentials}
35+
- configMapRef: {name: test-variables}
36+
queries:
37+
# Cluster metrics
38+
- query: kubernetes_build_info{cluster="$CLUSTER", job="integrations/kubernetes/kubelet", source="eks-addon"}
39+
type: promql
40+
- query: node_cpu_usage_seconds_total{cluster="$CLUSTER", job="integrations/kubernetes/resources", source="eks-addon"}
41+
type: promql
42+
- query: machine_memory_bytes{cluster="$CLUSTER", job="integrations/kubernetes/cadvisor", source="eks-addon"}
43+
type: promql
44+
- query: count(kube_node_info{cluster="$CLUSTER", job="integrations/kubernetes/kube-state-metrics", source="eks-addon"})
45+
type: promql
46+
47+
# Cluster events
48+
- query: count_over_time({cluster="$CLUSTER", job="integrations/kubernetes/eventhandler", source="eks-addon"}[1h])
49+
type: logql
50+
51+
# Pod logs
52+
- query: count_over_time({cluster="$CLUSTER", job!~"integrations/kubernetes/eventhandler|integrations/kubernetes/journal", source="eks-addon"}[1h])
53+
type: logql
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
---
2+
kind: ClusterConfig
3+
apiVersion: eksctl.io/v1alpha5
4+
metadata:
5+
region: ap-northeast-2
6+
tags:
7+
source: k8s-monitoring-helm-platform-test
8+
iam:
9+
withOIDC: true
10+
nodeGroups:
11+
- name: ng-linux
12+
instanceType: m5.large
13+
minSize: 1
14+
maxSize: 1

0 commit comments

Comments
 (0)