Skip to content

Commit c4103e0

Browse files
petewallpetewall
authored
Annotation Autodiscovery improvements: (#1305)
* Annotation Autodiscovery improvements: * Label selectors * Scrape request parameters * label and static label settings Signed-off-by: Pete Wall <[email protected]> * Add namespace label Signed-off-by: Pete Wall <[email protected]> --------- Signed-off-by: Pete Wall <[email protected]>
1 parent 45a95dc commit c4103e0

File tree

34 files changed

+2014
-345
lines changed

34 files changed

+2014
-345
lines changed

‎charts/k8s-monitoring/charts/feature-annotation-autodiscovery/README.md

+37
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ Be sure perform actual integration testing in a live environment in the main [k8
5757
|-----|------|---------|-------------|
5858
| annotations.instance | string | `"k8s.grafana.com/instance"` | Annotation for overriding the instance label |
5959
| annotations.job | string | `"k8s.grafana.com/job"` | Annotation for overriding the job label |
60+
| annotations.metricsParam | string | `"k8s.grafana.com/metrics.param"` | Annotation for setting __param_<key> parameters when scraping. Example: `k8s.grafana.com/metrics.param_key: "value"`. |
6061
| annotations.metricsPath | string | `"k8s.grafana.com/metrics.path"` | Annotation for setting or overriding the metrics path. If not set, it defaults to /metrics |
6162
| annotations.metricsPortName | string | `"k8s.grafana.com/metrics.portName"` | Annotation for setting the metrics port by name. |
6263
| annotations.metricsPortNumber | string | `"k8s.grafana.com/metrics.portNumber"` | Annotation for setting the metrics port by number. |
@@ -79,6 +80,7 @@ Be sure perform actual integration testing in a live environment in the main [k8
7980
|-----|------|---------|-------------|
8081
| excludeNamespaces | list | `[]` | The list of namespaces to exclude from autodiscovery. |
8182
| extraDiscoveryRules | string | `""` | Rule blocks to be added to the discovery.relabel component for discovered pods and services. These relabeling rules are applied pre-scrape against the targets from service discovery. Before the scrape, any remaining target labels that start with `__` (i.e. `__meta_kubernetes*`) are dropped. ([docs](https://grafana.com/docs/alloy/latest/reference/components/discovery/discovery.relabel/#rule-block)) |
83+
| labelSelectors | object | `{}` | Filter the list of discovered pods and services by labels. Example: `labelSelectors: { 'app': 'myapp' }` will only discover pods and services with the label `app=myapp`. Example: `labelSelectors: { 'app': ['myapp', 'myotherapp'] }` will only discover pods and services with the label `app=myapp` or `app=myotherapp`. |
8284
| namespaces | list | `[]` | The list of namespaces to include in autodiscovery. If empty, all namespaces are included. |
8385

8486
### Metric Processing Settings
@@ -104,3 +106,38 @@ Be sure perform actual integration testing in a live environment in the main [k8
104106
| global.maxCacheSize | int | `100000` | Sets the max_cache_size for every prometheus.relabel component. ([docs](https://grafana.com/docs/alloy/latest/reference/components/prometheus/prometheus.relabel/#arguments)) This should be at least 2x-5x your largest scrape target or samples appended rate. |
105107
| global.scrapeInterval | string | `"60s"` | How frequently to scrape metrics. |
106108
| global.scrapeTimeout | string | `"10s"` | The scrape timeout for discovered pods and services. |
109+
110+
### Pod Discovery Settings
111+
112+
| Key | Type | Default | Description |
113+
|-----|------|---------|-------------|
114+
| pods.enabled | bool | `true` | Enable discovering Pods with annotations. |
115+
| pods.labelSelectors | object | `{}` | Filter the list of discovered Pods by labels. Example: `labelSelectors: { 'app': 'myapp' }` will only discover Pods with the label `app=myapp`. Example: `labelSelectors: { 'app': ['myapp', 'myotherapp'] }` will only discover Pods with the label `app=myapp` or `app=myotherapp`. |
116+
117+
### Pod Metric Processing Settings
118+
119+
| Key | Type | Default | Description |
120+
|-----|------|---------|-------------|
121+
| pods.labels | object | `{}` | Add labels to metrics from discovered Pods. Runs during discovery, so __meta_ labels are available. See the [documentation](https://grafana.com/docs/alloy/latest/reference/components/discovery/discovery.kubernetes/#pod-role) for the full list of meta labels. |
122+
| pods.staticLabels | object | `{}` | Metric labels to set with static data for discovered Pods. |
123+
| pods.staticLabelsFrom | object | `{}` | Static labels to set on metrics from discovered Pods, not quoted so it can reference config components. |
124+
125+
### Services
126+
127+
| Key | Type | Default | Description |
128+
|-----|------|---------|-------------|
129+
| services.enabled | bool | `true` | Enable discovering Services with annotations. |
130+
131+
### Service Discovery Settings
132+
133+
| Key | Type | Default | Description |
134+
|-----|------|---------|-------------|
135+
| services.labelSelectors | object | `{}` | Filter the list of discovered Services by labels. Example: `labelSelectors: { 'app': 'myapp' }` will only discover Services with the label `app=myapp`. Example: `labelSelectors: { 'app': ['myapp', 'myotherapp'] }` will only discover Services with the label `app=myapp` or `app=myotherapp`. |
136+
137+
### Service Metric Processing Settings
138+
139+
| Key | Type | Default | Description |
140+
|-----|------|---------|-------------|
141+
| services.labels | object | `{}` | Add labels to metrics from discovered Services. Run during discovery, so __meta_ labels are available. See the [documentation](https://grafana.com/docs/alloy/latest/reference/components/discovery/discovery.kubernetes/#service-role) for the full list of meta labels. |
142+
| services.staticLabels | object | `{}` | Metric labels to set with static data for discovered Services. |
143+
| services.staticLabelsFrom | object | `{}` | Static labels to set on metrics from discovered Services, not quoted so it can reference config components. |

‎charts/k8s-monitoring/charts/feature-annotation-autodiscovery/templates/_helpers.tpl

+11-3
Original file line numberDiff line numberDiff line change
@@ -16,14 +16,22 @@ If release name contains chart name it will be used as a full name.
1616
{{- end }}
1717
{{- end }}
1818

19-
{{- define "escape_annotation" -}}
19+
{{- define "escape_label_or_annotation" -}}
2020
{{ . | replace "-" "_" | replace "." "_" | replace "/" "_" }}
2121
{{- end }}
2222

2323
{{- define "pod_annotation" -}}
24-
{{ printf "__meta_kubernetes_pod_annotation_%s" (include "escape_annotation" .) }}
24+
{{ printf "__meta_kubernetes_pod_annotation_%s" (include "escape_label_or_annotation" .) }}
25+
{{- end }}
26+
27+
{{- define "pod_label" -}}
28+
{{ printf "__meta_kubernetes_pod_label_%s" (include "escape_label_or_annotation" .) }}
2529
{{- end }}
2630

2731
{{- define "service_annotation" -}}
28-
{{ printf "__meta_kubernetes_service_annotation_%s" (include "escape_annotation" .) }}
32+
{{ printf "__meta_kubernetes_service_annotation_%s" (include "escape_label_or_annotation" .) }}
33+
{{- end }}
34+
35+
{{- define "service_label" -}}
36+
{{ printf "__meta_kubernetes_service_label_%s" (include "escape_label_or_annotation" .) }}
2937
{{- end }}

‎charts/k8s-monitoring/charts/feature-annotation-autodiscovery/templates/_module.alloy.tpl

+49-217
Original file line numberDiff line numberDiff line change
@@ -3,223 +3,12 @@ declare "annotation_autodiscovery" {
33
argument "metrics_destinations" {
44
comment = "Must be a list of metric destinations where collected metrics should be forwarded to"
55
}
6-
7-
discovery.kubernetes "pods" {
8-
role = "pod"
9-
{{- if .Values.namespaces }}
10-
namespaces {
11-
names = {{ .Values.namespaces | toJson }}
12-
}
13-
{{- end }}
14-
}
15-
16-
discovery.relabel "annotation_autodiscovery_pods" {
17-
targets = discovery.kubernetes.pods.targets
18-
{{- if .Values.excludeNamespaces }}
19-
rule {
20-
source_labels = ["__meta_kubernetes_namespace"]
21-
regex = "{{ join "|" .Values.excludeNamespaces }}"
22-
action = "drop"
23-
}
24-
{{- end }}
25-
rule {
26-
source_labels = ["{{ include "pod_annotation" .Values.annotations.scrape }}"]
27-
regex = "true"
28-
action = "keep"
29-
}
30-
rule {
31-
source_labels = ["{{ include "pod_annotation" .Values.annotations.job }}"]
32-
action = "replace"
33-
target_label = "job"
34-
}
35-
rule {
36-
source_labels = ["{{ include "pod_annotation" .Values.annotations.instance }}"]
37-
action = "replace"
38-
target_label = "instance"
39-
}
40-
rule {
41-
source_labels = ["{{ include "pod_annotation" .Values.annotations.metricsPath }}"]
42-
action = "replace"
43-
target_label = "__metrics_path__"
44-
}
45-
46-
// Choose the pod port
47-
// The discovery generates a target for each declared container port of the pod.
48-
// If the metricsPortName annotation has value, keep only the target where the port name matches the one of the annotation.
49-
rule {
50-
source_labels = ["__meta_kubernetes_pod_container_port_name"]
51-
target_label = "__tmp_port"
52-
}
53-
rule {
54-
source_labels = ["{{ include "pod_annotation" .Values.annotations.metricsPortName }}"]
55-
regex = "(.+)"
56-
target_label = "__tmp_port"
57-
}
58-
rule {
59-
source_labels = ["__meta_kubernetes_pod_container_port_name"]
60-
action = "keepequal"
61-
target_label = "__tmp_port"
62-
}
63-
rule {
64-
action = "labeldrop"
65-
regex = "__tmp_port"
66-
}
67-
68-
// If the metrics port number annotation has a value, override the target address to use it, regardless whether it is
69-
// one of the declared ports on that Pod.
70-
rule {
71-
source_labels = ["{{ include "pod_annotation" .Values.annotations.metricsPortNumber }}", "__meta_kubernetes_pod_ip"]
72-
regex = "(\\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4})"
73-
replacement = "[$2]:$1" // IPv6
74-
target_label = "__address__"
75-
}
76-
rule {
77-
source_labels = ["{{ include "pod_annotation" .Values.annotations.metricsPortNumber }}", "__meta_kubernetes_pod_ip"]
78-
regex = "(\\d+);((([0-9]+?)(\\.|$)){4})" // IPv4, takes priority over IPv6 when both exists
79-
replacement = "$2:$1"
80-
target_label = "__address__"
81-
}
82-
83-
rule {
84-
source_labels = ["{{ include "pod_annotation" .Values.annotations.metricsScheme }}"]
85-
regex = "(.+)"
86-
target_label = "__scheme__"
87-
}
88-
89-
rule {
90-
source_labels = ["{{ include "pod_annotation" .Values.annotations.metricsScrapeInterval }}"]
91-
regex = "(.+)"
92-
target_label = "__scrape_interval__"
93-
}
94-
rule {
95-
source_labels = ["__scrape_interval__"]
96-
regex = ""
97-
replacement = {{ .Values.scrapeInterval | default .Values.global.scrapeInterval | quote }}
98-
target_label = "__scrape_interval__"
99-
}
100-
rule {
101-
source_labels = ["{{ include "pod_annotation" .Values.annotations.metricsScrapeTimeout }}"]
102-
regex = "(.+)"
103-
target_label = "__scrape_timeout__"
104-
}
105-
rule {
106-
source_labels = ["__scrape_timeout__"]
107-
regex = ""
108-
replacement = {{ .Values.scrapeTimeout | default .Values.global.scrapeTimeout | quote }}
109-
target_label = "__scrape_timeout__"
110-
}
111-
{{- if .Values.extraDiscoveryRules }}
112-
{{ .Values.extraDiscoveryRules | indent 4 }}
113-
{{- end }}
114-
}
115-
116-
discovery.kubernetes "services" {
117-
role = "service"
118-
{{- if .Values.namespaces }}
119-
namespaces {
120-
names = {{ .Values.namespaces | toJson }}
121-
}
6+
{{- if .Values.pods.enabled }}
7+
{{- include "feature.annotationAutodiscovery.pods" . | indent 2 }}
1228
{{- end }}
123-
}
124-
125-
discovery.relabel "annotation_autodiscovery_services" {
126-
targets = discovery.kubernetes.services.targets
127-
{{- if .Values.excludeNamespaces }}
128-
rule {
129-
source_labels = ["__meta_kubernetes_namespace"]
130-
regex = "{{ join "|" .Values.excludeNamespaces }}"
131-
action = "drop"
132-
}
133-
{{- end }}
134-
rule {
135-
source_labels = ["{{ include "service_annotation" .Values.annotations.scrape }}"]
136-
regex = "true"
137-
action = "keep"
138-
}
139-
rule {
140-
source_labels = ["{{ include "service_annotation" .Values.annotations.job }}"]
141-
action = "replace"
142-
target_label = "job"
143-
}
144-
rule {
145-
source_labels = ["{{ include "service_annotation" .Values.annotations.instance }}"]
146-
action = "replace"
147-
target_label = "instance"
148-
}
149-
rule {
150-
source_labels = ["{{ include "service_annotation" .Values.annotations.metricsPath }}"]
151-
action = "replace"
152-
target_label = "__metrics_path__"
153-
}
154-
155-
// Choose the service port
156-
rule {
157-
source_labels = ["__meta_kubernetes_service_port_name"]
158-
target_label = "__tmp_port"
159-
}
160-
rule {
161-
source_labels = ["{{ include "service_annotation" .Values.annotations.metricsPortName }}"]
162-
regex = "(.+)"
163-
target_label = "__tmp_port"
164-
}
165-
rule {
166-
source_labels = ["__meta_kubernetes_service_port_name"]
167-
action = "keepequal"
168-
target_label = "__tmp_port"
169-
}
170-
171-
rule {
172-
source_labels = ["__meta_kubernetes_service_port_number"]
173-
target_label = "__tmp_port"
174-
}
175-
rule {
176-
source_labels = ["{{ include "service_annotation" .Values.annotations.metricsPortNumber }}"]
177-
regex = "(.+)"
178-
target_label = "__tmp_port"
179-
}
180-
rule {
181-
source_labels = ["__meta_kubernetes_service_port_number"]
182-
action = "keepequal"
183-
target_label = "__tmp_port"
184-
}
185-
rule {
186-
action = "labeldrop"
187-
regex = "__tmp_port"
188-
}
189-
190-
rule {
191-
source_labels = ["{{ include "service_annotation" .Values.annotations.metricsScheme }}"]
192-
regex = "(.+)"
193-
target_label = "__scheme__"
194-
}
195-
196-
rule {
197-
source_labels = ["{{ include "service_annotation" .Values.annotations.metricsScrapeInterval }}"]
198-
regex = "(.+)"
199-
target_label = "__scrape_interval__"
200-
}
201-
rule {
202-
source_labels = ["__scrape_interval__"]
203-
regex = ""
204-
replacement = {{ .Values.scrapeInterval | default .Values.global.scrapeInterval | quote }}
205-
target_label = "__scrape_interval__"
206-
}
207-
rule {
208-
source_labels = ["{{ include "service_annotation" .Values.annotations.metricsScrapeTimeout }}"]
209-
regex = "(.+)"
210-
target_label = "__scrape_timeout__"
211-
}
212-
rule {
213-
source_labels = ["__scrape_timeout__"]
214-
regex = ""
215-
replacement = {{ .Values.scrapeTimeout | default .Values.global.scrapeTimeout | quote }}
216-
target_label = "__scrape_timeout__"
217-
}
218-
219-
{{- if .Values.extraDiscoveryRules }}
220-
{{ .Values.extraDiscoveryRules | indent 4 }}
9+
{{- if .Values.services.enabled }}
10+
{{- include "feature.annotationAutodiscovery.services" . | indent 2 }}
22111
{{- end }}
222-
}
22312

22413
discovery.relabel "annotation_autodiscovery_http" {
22514
targets = array.concat(discovery.relabel.annotation_autodiscovery_pods.output, discovery.relabel.annotation_autodiscovery_services.output)
@@ -248,7 +37,10 @@ declare "annotation_autodiscovery" {
24837
clustering {
24938
enabled = true
25039
}
251-
{{ if or .Values.metricsTuning.includeMetrics .Values.metricsTuning.excludeMetrics .Values.extraMetricProcessingRules }}
40+
{{- $metricRelabelRulesNeeded := or .Values.metricsTuning.includeMetrics .Values.metricsTuning.excludeMetrics .Values.extraMetricProcessingRules }}
41+
{{- $metricRelabelRulesNeeded = or $metricRelabelRulesNeeded (and .Values.pods.enabled (or .Values.pods.staticLabels .Values.pods.staticLabelsFrom)) }}
42+
{{- $metricRelabelRulesNeeded = or $metricRelabelRulesNeeded (and .Values.services.enabled (or .Values.services.staticLabels .Values.services.staticLabelsFrom)) }}
43+
{{ if $metricRelabelRulesNeeded }}
25244
forward_to = [prometheus.relabel.annotation_autodiscovery.receiver]
25345
{{- else }}
25446
forward_to = argument.metrics_destinations.value
@@ -267,7 +59,7 @@ declare "annotation_autodiscovery" {
26759
clustering {
26860
enabled = true
26961
}
270-
{{ if or .Values.metricsTuning.includeMetrics .Values.metricsTuning.excludeMetrics .Values.extraMetricProcessingRules }}
62+
{{ if $metricRelabelRulesNeeded }}
27163
forward_to = [prometheus.relabel.annotation_autodiscovery.receiver]
27264
}
27365

@@ -287,6 +79,46 @@ declare "annotation_autodiscovery" {
28779
action = "drop"
28880
}
28981
{{- end }}
82+
{{- if .Values.pods.enabled }}
83+
{{- range $k, $v := .Values.pods.staticLabels }}
84+
rule {
85+
source_labels = ["temp_source"]
86+
regex = "pod"
87+
target_label = {{ $k | quote }}
88+
replacement = {{ $v | quote }}
89+
}
90+
{{- end }}
91+
{{- range $k, $v := .Values.pods.staticLabelsFrom }}
92+
rule {
93+
source_labels = ["temp_source"]
94+
regex = "pod"
95+
target_label = {{ $k | quote }}
96+
replacement = {{ $v }}
97+
}
98+
{{- end }}
99+
{{- end }}
100+
{{- if .Values.services.enabled }}
101+
{{- range $k, $v := .Values.services.staticLabels }}
102+
rule {
103+
source_labels = ["temp_source"]
104+
regex = "service"
105+
target_label = {{ $k | quote }}
106+
replacement = {{ $v | quote }}
107+
}
108+
{{- end }}
109+
{{- range $k, $v := .Values.services.staticLabelsFrom }}
110+
rule {
111+
source_labels = ["temp_source"]
112+
regex = "service"
113+
target_label = {{ $k | quote }}
114+
replacement = {{ $v }}
115+
}
116+
{{- end }}
117+
{{- end }}
118+
rule {
119+
action = "labeldrop"
120+
regex = "temp_source"
121+
}
290122
{{- if .Values.extraMetricProcessingRules }}
291123
{{ .Values.extraMetricProcessingRules | indent 4 }}
292124
{{- end }}

0 commit comments

Comments
 (0)