Add the ability to specify processing stages per loki destination (#1186)

Provide an integration test that shows how to use them

Signed-off-by: Pete Wall <[email protected]>

Author: petewall

charts/k8s-monitoring/destinations/loki-values.yaml

@@ -43,6 +43,11 @@ extraLabels: {}
 # @section -- General
 extraLabelsFrom: {}
 
+# -- Stage blocks to be evaluated before delivering to the Loki destination. See
+# ([docs](https://grafana.com/docs/alloy/latest/reference/components/loki/loki.process/#blocks)) for more information.
+# @section -- General
+logProcessingStages: ""
+
 auth:
   # -- The type of authentication to do.
   # Options are "none" (default), "basic", "bearerToken", "oauth2".

charts/k8s-monitoring/docs/destinations/loki.md

@@ -57,6 +57,7 @@ This defines the options for defining a destination for logs that use the Loki p
 | extraHeadersFrom | object | `{}` | Extra headers to be set when sending data through a dynamic reference. All values are treated as raw strings and not quoted. |
 | extraLabels | object | `{}` | Custom labels to be added to all logs and events. All values are treated as strings and automatically quoted. |
 | extraLabelsFrom | object | `{}` | Custom labels to be added to all logs and events through a dynamic reference. All values are treated as raw strings and not quoted. |
+| logProcessingStages | string | `""` | Stage blocks to be evaluated before delivering to the Loki destination. See ([docs](https://grafana.com/docs/alloy/latest/reference/components/loki/loki.process/#blocks)) for more information. |
 | name | string | `""` | The name for this Loki destination. |
 | proxyURL | string | `""` | The Proxy URL for the Loki destination. |
 | tenantId | string | `""` | The tenant ID for the Loki destination. |

charts/k8s-monitoring/schema-mods/definitions/loki-destination.schema.json

@@ -98,6 +98,9 @@
         "extraLabelsFrom": {
             "type": "object"
         },
+        "logProcessingStages": {
+            "type": "string"
+        },
         "name": {
             "type": "string"
         },

charts/k8s-monitoring/templates/destinations/_destination_loki.tpl

@@ -2,8 +2,15 @@
 {{- $defaultValues := "destinations/loki-values.yaml" | .Files.Get | fromYaml }}
 {{- with merge .destination $defaultValues }}
 otelcol.exporter.loki {{ include "helper.alloy_name" .name | quote }} {
+  forward_to = [{{ include "destinations.loki.alloy.loki.logs.target" . }}]
+}
+{{- if .logProcessingStages }}
+
+loki.process {{ include "helper.alloy_name" .name | quote }} {
+{{ .logProcessingStages | indent 2 }}
   forward_to = [loki.write.{{ include "helper.alloy_name" .name }}.receiver]
 }
+{{- end }}
 
 loki.write {{ include "helper.alloy_name" .name | quote }} {
   endpoint {
@@ -122,7 +129,13 @@ loki.write {{ include "helper.alloy_name" .name | quote }} {
 - tls.key
 {{- end -}}
 
-{{- define "destinations.loki.alloy.loki.logs.target" }}loki.write.{{ include "helper.alloy_name" .name }}.receiver{{ end -}}
+{{- define "destinations.loki.alloy.loki.logs.target" }}
+{{- if .logProcessingStages -}}
+loki.process.{{ include "helper.alloy_name" .name }}.receiver
+{{- else -}}
+loki.write.{{ include "helper.alloy_name" .name }}.receiver
+{{- end -}}
+{{- end -}}
 {{- define "destinations.loki.alloy.otlp.logs.target" }}otelcol.exporter.loki.{{ include "helper.alloy_name" .name }}.input{{ end -}}
 
 {{- define "destinations.loki.supports_metrics" }}false{{ end -}}