Skip to content

Commit 0e6fc87

Browse files
petewallpetewall
committed
WIP:
* make it so all log gathering methods are possible at once. * Work on the openshift log gethering Signed-off-by: Pete Wall <pete.wall@grafana.com>
1 parent 1e19939 commit 0e6fc87

11 files changed

Lines changed: 365 additions & 121 deletions

File tree

‎charts/k8s-monitoring/charts/feature-pod-logs/templates/_api.alloy.tpl‎

Lines changed: 140 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,150 @@
11
{{- define "feature.podLogs.kubernetesApi.alloy" }}
2-
discovery.kubernetes "pods" {
2+
discovery.kubernetes "kubernetes_api_pods" {
33
role = "pod"
4-
{{- if .Values.namespaces }}
4+
{{- if .Values.kubernetesApiGathering.namespaces }}
55
namespaces {
66
names = {{ .Values.namespaces | toJson }}
77
}
88
{{- end }}
9+
{{- if or .Values.kubernetesApiGathering.labelSelectors .Values.kubernetesApiGathering.fieldSelectors }}
10+
selectors {
11+
role = "pod"
12+
{{- if .Values.kubernetesApiGathering.labelSelectors }}
13+
label = {{ .Values.kubernetesApiGathering.labelSelectors | toJson }}
14+
{{- end }}
15+
{{- if .Values.kubernetesApiGathering.fieldSelectors }}
16+
field = {{ .Values.kubernetesApiGathering.fieldSelectors | join "," | quote }}
17+
{{- end }}
18+
}
19+
{{- end }}
20+
{{- if or .Values.kubernetesApiGathering.nodeLabelSelectors .Values.kubernetesApiGathering.nodeFieldSelectors }}
21+
attach_metadata {
22+
node = true
23+
}
24+
selectors {
25+
role = "node"
26+
{{- if .Values.kubernetesApiGathering.nodeLabelSelectors }}
27+
label = {{ .Values.kubernetesApiGathering.nodeLabelSelectors | toJson }}
28+
{{- end }}
29+
{{- if .Values.kubernetesApiGathering.nodeFieldSelectors }}
30+
field = {{ .Values.kubernetesApiGathering.nodeFieldSelectors | join "," | quote }}
31+
{{- end }}
32+
}
33+
{{- end }}
34+
}
35+
36+
discovery.relabel "kubernetes_api_pods" {
37+
targets = discovery.kubernetes.kubernetes_api_pods.targets
38+
rule {
39+
source_labels = ["__meta_kubernetes_namespace"]
40+
action = "replace"
41+
target_label = "namespace"
42+
}
43+
{{- if .Values.kubernetesApiGathering.excludeNamespaces }}
44+
rule {
45+
source_labels = ["namespace"]
46+
regex = "{{ .Values.kubernetesApiGathering.excludeNamespaces | join "|" }}"
47+
action = "drop"
48+
}
49+
{{- end }}
50+
rule {
51+
source_labels = ["__meta_kubernetes_pod_name"]
52+
target_label = "pod"
53+
}
54+
rule {
55+
source_labels = ["__meta_kubernetes_pod_container_name"]
56+
target_label = "container"
57+
}
58+
59+
rule {
60+
source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_container_name"]
61+
separator = "/"
62+
replacement = "$1"
63+
target_label = "job"
64+
}
65+
66+
// set the container runtime as a label
67+
rule {
68+
source_labels = ["__meta_kubernetes_pod_container_id"]
69+
regex = "^(\\S+):\\/\\/.+$"
70+
target_label = "tmp_container_runtime"
71+
}
72+
73+
// make all labels on the pod available to the pipeline as labels,
74+
// they are omitted before write to loki via stage.label_keep unless explicitly set
75+
rule {
76+
action = "labelmap"
77+
regex = "__meta_kubernetes_pod_label_(.+)"
78+
}
79+
80+
// make all annotations on the pod available to the pipeline as labels,
81+
// they are omitted before write to loki via stage.label_keep unless explicitly set
82+
rule {
83+
action = "labelmap"
84+
regex = "__meta_kubernetes_pod_annotation_(.+)"
85+
}
86+
87+
{{- range $label, $k8sAnnotation := .Values.annotations }}
88+
rule {
89+
source_labels = ["{{ include "pod_annotation" $k8sAnnotation }}"]
90+
target_label = {{ $label | quote }}
91+
}
92+
{{- end }}
93+
{{- range $label, $k8sLabels := .Values.labels }}
94+
rule {
95+
source_labels = ["{{ include "pod_label" $k8sLabels }}"]
96+
target_label = {{ $label | quote }}
97+
}
98+
{{- end }}
99+
100+
// explicitly set service_name. if not set, loki will automatically try to populate a default.
101+
// see https://grafana.com/docs/loki/latest/get-started/labels/#default-labels-for-all-users
102+
//
103+
// choose the first value found from the following ordered list:
104+
// - pod.annotation[resource.opentelemetry.io/service.name]
105+
// - pod.label[app.kubernetes.io/name]
106+
// - k8s.pod.name
107+
// - k8s.container.name
108+
rule {
109+
action = "replace"
110+
source_labels = [
111+
"__meta_kubernetes_pod_annotation_resource_opentelemetry_io_service_name",
112+
"__meta_kubernetes_pod_label_app_kubernetes_io_name",
113+
"__meta_kubernetes_pod_name",
114+
"__meta_kubernetes_pod_container_name",
115+
]
116+
separator = ";"
117+
regex = "^(?:;*)?([^;]+).*$"
118+
replacement = "$1"
119+
target_label = "service_name"
120+
}
121+
122+
// set service_namespace
123+
rule {
124+
action = "replace"
125+
source_labels = ["__meta_kubernetes_pod_annotation_resource_opentelemetry_io_service_namespace"]
126+
target_label = "service_namespace"
127+
}
128+
129+
// set deployment_environment and deployment_environment_name
130+
rule {
131+
action = "replace"
132+
source_labels = ["__meta_kubernetes_pod_annotation_resource_opentelemetry_io_deployment_environment_name"]
133+
target_label = "deployment_environment_name"
134+
}
135+
rule {
136+
action = "replace"
137+
source_labels = ["__meta_kubernetes_pod_annotation_resource_opentelemetry_io_deployment_environment"]
138+
target_label = "deployment_environment"
139+
}
140+
141+
{{- if .Values.kubernetesApiGathering.extraDiscoveryRules }}
142+
{{ .Values.kubernetesApiGathering.extraDiscoveryRules | indent 2 }}
143+
{{- end }}
9144
}
10145

11-
loki.source.kubernetes "pod_logs" {
12-
targets = discovery.relabel.filtered_pods.output
13-
forward_to = [loki.process.pod_logs.receiver]
146+
loki.source.kubernetes "kubernetes_api_pods" {
147+
targets = discovery.relabel.kubernetes_api_pods.output
148+
forward_to = [loki.process.pod_log_processor.receiver]
14149
}
15150
{{- end }}

‎charts/k8s-monitoring/charts/feature-pod-logs/templates/_collector_validation.tpl‎

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,16 @@
11
{{/* Validates that the Alloy instance is appropriate for the given Pod Logs settings */}}
22
{{/* Inputs: Values (Pod Logs values), Collector (Alloy values), CollectorName (string) */}}
33
{{- define "feature.podLogs.collector.validate" -}}
4-
{{- if eq .Values.gatherMethod "volumes" }}
4+
{{- if or .Values.volumeGathering.enabled (eq .Values.gatherMethod "volumes") }}
55
{{- if not (eq .Collector.controller.type "daemonset") }}
66
{{- fail (printf "Pod Logs feature requires Alloy to be a DaemonSet when using the \"volumes\" gather method.\nPlease set:\n%s:\n controller:\n type: daemonset" .CollectorName) }}
77
{{- end -}}
88
{{- if not .Collector.alloy.mounts.varlog }}
99
{{- fail (printf "Pod Logs feature requires Alloy to mount /var/log when using the \"volumes\" gather method.\nPlease set:\n%s:\n alloy:\n mounts:\n varlog: true" .CollectorName) }}
1010
{{- end -}}
11-
{{- if .Collector.alloy.clustering.enabled }}
12-
{{- fail (printf "Pod Logs feature requires Alloy clustering to be disabled when using the \"volumes\" gather method.\nPlease set:\n%s:\n alloy:\n clustering:\n enabled: false" .CollectorName) }}
13-
{{- end -}}
14-
{{- else if eq .Values.gatherMethod "kubernetesApi" }}
15-
{{- if .Collector.alloy.mounts.varlog }}
16-
{{- fail (printf "Pod Logs feature should not mount /var/log when using the \"kubernetesApi\" gather method.\nPlease set:\n%s:\n alloy:\n mounts:\n varlog: false" .CollectorName) }}
17-
{{- end -}}
11+
{{- end -}}
12+
13+
{{- if or .Values.kubernetesApiGathering.enabled (eq .Values.gatherMethod "kubernetesApi") }}
1814
{{- if not .Collector.alloy.clustering.enabled }}
1915
{{- if eq .Collector.controller.type "daemonset" }}
2016
{{- fail (printf "Pod Logs feature requires Alloy DaemonSet to be in clustering mode when using the \"kubernetesApi\" gather method.\nPlease set:\n%s:\n alloy:\n clustering:\n enabled: true" .CollectorName) }}
@@ -23,4 +19,12 @@
2319
{{- end -}}
2420
{{- end -}}
2521
{{- end -}}
22+
23+
{{- if .Values.lokiReceiver.enabled }}
24+
{{- if not .Collector.alloy.clustering.enabled }}
25+
{{- fail (printf "Pod Logs feature requires Alloy to be in clustering mode when using the \"kubernetesApi\" gather method.\nPlease set:\n%s:\n alloy:\n clustering:\n enabled: true" .CollectorName) }}
26+
{{- end -}}
27+
{{- end -}}
28+
29+
2630
{{- end -}}

‎charts/k8s-monitoring/charts/feature-pod-logs/templates/_common_log_processing.alloy.tpl‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{{- define "feature.podLogs.processing.alloy" }}
2-
loki.process "pod_logs" {
2+
loki.process "pod_log_processor" {
33
stage.match {
44
selector = "{tmp_container_runtime=~\"containerd|cri-o\"}"
55
// the cri processing stage extracts the following k/v pairs: log, stream, time, flags

‎charts/k8s-monitoring/charts/feature-pod-logs/templates/_log_receiver.alloy.tpl‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
1-
{{- define "feature.podLogs.logReceiver.alloy" }}
2-
loki.source.api "logs_receiver" {
1+
{{- define "feature.podLogs.lokiReceiver.alloy" }}
2+
loki.source.api "loki_receiver" {
33
http {
44
listen_address = "0.0.0.0"
5-
listen_port = 3100
5+
listen_port = {{ .Values.lokiReceiver.port }}
66
}
77

8-
forward_to = [loki.process.pod_logs.receiver]
8+
forward_to = [loki.process.pod_log_processor.receiver]
99
}
1010
{{- end }}

‎charts/k8s-monitoring/charts/feature-pod-logs/templates/_module.alloy.tpl‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,16 +4,16 @@ declare "pod_logs" {
44
comment = "Must be a list of log destinations where collected logs should be forwarded to"
55
}
66

7-
{{- if or (eq .Values.gatherMethod "volumes") (eq .Values.gatherMethod "kubernetesApi") }}
8-
{{- include "feature.podLogs.discovery.alloy" . | nindent 2 }}
7+
{{- if .Values.volumeGathering.enabled }}
8+
{{- include "feature.podLogs.volumeGathering.alloy" . | nindent 2 }}
99
{{- end }}
1010

11-
{{- if eq .Values.gatherMethod "volumes" }}
12-
{{- include "feature.podLogs.volumes.alloy" . | nindent 2 }}
13-
{{- else if eq .Values.gatherMethod "kubernetesApi" }}
11+
{{- if .Values.kubernetesApiGathering.enabled }}
1412
{{- include "feature.podLogs.kubernetesApi.alloy" . | nindent 2 }}
15-
{{- else if eq .Values.gatherMethod "OpenShiftClusterLogForwarder" }}
16-
{{- include "feature.podLogs.logReceiver.alloy" . | nindent 2 }}
13+
{{- end }}
14+
15+
{{- if .Values.lokiReceiver.enabled }}
16+
{{- include "feature.podLogs.lokiReceiver.alloy" . | nindent 2 }}
1717
{{- end }}
1818

1919
{{- include "feature.podLogs.processing.alloy" . | nindent 2 }}

charts/k8s-monitoring/charts/feature-pod-logs/templates/_common_pod_discovery.alloy.tpl renamed to charts/k8s-monitoring/charts/feature-pod-logs/templates/_volumeGathering.alloy.tpl

Lines changed: 78 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,42 +1,76 @@
1-
{{- define "feature.podLogs.discovery.alloy" }}
2-
discovery.relabel "filtered_pods" {
3-
targets = discovery.kubernetes.pods.targets
1+
{{- define "feature.podLogs.volumeGathering.alloy" }}
2+
discovery.kubernetes "volume_gathering_pods" {
3+
role = "pod"
4+
selectors {
5+
role = "pod"
6+
field = "spec.nodeName=" + sys.env("HOSTNAME")
7+
}
8+
{{- if .Values.volumeGathering.namespaces }}
9+
namespaces {
10+
names = {{ .Values.namespaces | toJson }}
11+
}
12+
{{- end }}
13+
{{- if or .Values.volumeGathering.labelSelectors .Values.volumeGathering.fieldSelectors }}
14+
selectors {
15+
role = "pod"
16+
{{- if .Values.volumeGathering.labelSelectors }}
17+
label = {{ .Values.volumeGathering.labelSelectors | toJson }}
18+
{{- end }}
19+
{{- if .Values.volumeGathering.fieldSelectors }}
20+
field = {{ .Values.volumeGathering.fieldSelectors | join "," | quote }}
21+
{{- end }}
22+
}
23+
{{- end }}
24+
{{- if or .Values.volumeGathering.nodeLabelSelectors .Values.volumeGathering.nodeFieldSelectors }}
25+
attach_metadata {
26+
node = true
27+
}
28+
selectors {
29+
role = "node"
30+
{{- if .Values.volumeGathering.nodeLabelSelectors }}
31+
label = {{ .Values.volumeGathering.nodeLabelSelectors | toJson }}
32+
{{- end }}
33+
{{- if .Values.volumeGathering.nodeFieldSelectors }}
34+
field = {{ .Values.volumeGathering.nodeFieldSelectors | join "," | quote }}
35+
{{- end }}
36+
}
37+
{{- end }}
38+
}
39+
40+
discovery.relabel "volume_gathering_pods" {
41+
targets = discovery.kubernetes.volume_gathering_pods.targets
442
rule {
543
source_labels = ["__meta_kubernetes_namespace"]
644
action = "replace"
745
target_label = "namespace"
846
}
9-
{{- if .Values.excludeNamespaces }}
47+
{{- if .Values.volumeGathering.excludeNamespaces }}
1048
rule {
1149
source_labels = ["namespace"]
12-
regex = "{{ .Values.excludeNamespaces | join "|" }}"
50+
regex = "{{ .Values.volumeGathering.excludeNamespaces | join "|" }}"
1351
action = "drop"
1452
}
1553
{{- end }}
1654
rule {
1755
source_labels = ["__meta_kubernetes_pod_name"]
18-
action = "replace"
1956
target_label = "pod"
2057
}
2158
rule {
2259
source_labels = ["__meta_kubernetes_pod_container_name"]
23-
action = "replace"
2460
target_label = "container"
2561
}
62+
2663
rule {
2764
source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_container_name"]
2865
separator = "/"
29-
action = "replace"
3066
replacement = "$1"
3167
target_label = "job"
3268
}
3369

3470
// set the container runtime as a label
3571
rule {
36-
action = "replace"
3772
source_labels = ["__meta_kubernetes_pod_container_id"]
3873
regex = "^(\\S+):\\/\\/.+$"
39-
replacement = "$1"
4074
target_label = "tmp_container_runtime"
4175
}
4276

@@ -54,6 +88,19 @@ discovery.relabel "filtered_pods" {
5488
regex = "__meta_kubernetes_pod_annotation_(.+)"
5589
}
5690

91+
{{- range $label, $k8sAnnotation := .Values.annotations }}
92+
rule {
93+
source_labels = ["{{ include "pod_annotation" $k8sAnnotation }}"]
94+
target_label = {{ $label | quote }}
95+
}
96+
{{- end }}
97+
{{- range $label, $k8sLabels := .Values.labels }}
98+
rule {
99+
source_labels = ["{{ include "pod_label" $k8sLabels }}"]
100+
target_label = {{ $label | quote }}
101+
}
102+
{{- end }}
103+
57104
// explicitly set service_name. if not set, loki will automatically try to populate a default.
58105
// see https://grafana.com/docs/loki/latest/get-started/labels/#default-labels-for-all-users
59106
//
@@ -95,6 +142,13 @@ discovery.relabel "filtered_pods" {
95142
target_label = "deployment_environment"
96143
}
97144

145+
rule {
146+
source_labels = ["__meta_kubernetes_pod_uid", "__meta_kubernetes_pod_container_name"]
147+
separator = "/"
148+
replacement = "{{ .Values.volumeGathering.podLogsPath }}/*$1/*.log"
149+
target_label = "__path__"
150+
}
151+
98152
{{- range $label, $k8sAnnotation := .Values.annotations }}
99153
rule {
100154
source_labels = ["{{ include "pod_annotation" $k8sAnnotation }}"]
@@ -110,8 +164,20 @@ discovery.relabel "filtered_pods" {
110164
}
111165
{{- end }}
112166

113-
{{- if .Values.extraDiscoveryRules }}
114-
{{ .Values.extraDiscoveryRules | indent 2 }}
167+
{{- if .Values.volumeGathering.extraDiscoveryRules }}
168+
{{ .Values.volumeGathering.extraDiscoveryRules | indent 2 }}
115169
{{- end }}
116170
}
171+
172+
local.file_match "volume_gathering_pods" {
173+
path_targets = discovery.relabel.volume_gathering_pods.output
174+
}
175+
176+
loki.source.file "volume_gathering_pods" {
177+
targets = local.file_match.volume_gathering_pods.targets
178+
{{- if .Values.volumeGathering.onlyGatherNewLogLines | default .Values.volumeGatherSettings.onlyGatherNewLogLines }}
179+
tail_from_end = {{ .Values.volumeGathering.onlyGatherNewLogLines | default .Values.volumeGatherSettings.onlyGatherNewLogLines }}
117180
{{- end }}
181+
forward_to = [loki.process.pod_log_processor.receiver]
182+
}
183+
{{- end -}}

0 commit comments

Comments
 (0)