LOG-6860: Support Splunk Metadata keys in ClusterLogForwarder

Signed-off-by: Vitalii Parfonov <[email protected]>

Author: vparfonov

api/observability/v1/clusterlogforwarder_types.go

@@ -344,6 +344,15 @@ type ClusterLogForwarderList struct {
 	Items           []ClusterLogForwarder `json:"items"`
 }
 
+// FieldPath represents a path to find a value for a given field.  The format must be a value that can be converted to a
+// valid collector configuration. It is a dot delimited path to a field in the log record. It must start with a `.`.
+// The path can contain alphanumeric characters and underscores (a-zA-Z0-9_).
+// If segments contain characters outside of this range, the segment must be quoted.
+// Examples: `.kubernetes.namespace_name`, `.log_type`, '.kubernetes.labels.foobar', `.kubernetes.labels."foo-bar/baz"`
+//
+// +kubebuilder:validation:Pattern:=`^(\.[a-zA-Z0-9_]+|\."[^"]+")(\.[a-zA-Z0-9_]+|\."[^"]+")*$`
+type FieldPath string
+
 func init() {
 	SchemeBuilder.Register(&ClusterLogForwarder{}, &ClusterLogForwarderList{})
 }

api/observability/v1/filter_types.go

@@ -40,15 +40,6 @@ var (
 	}
 )
 
-// FieldPath represents a path to find a value for a given field.  The format must a value that can be converted to a
-// valid collector configuration. It is a dot delimited path to a field in the log record. It must start with a `.`.
-// The path can contain alphanumeric characters and underscores (a-zA-Z0-9_).
-// If segments contain characters outside of this range, the segment must be quoted.
-// Examples: `.kubernetes.namespace_name`, `.log_type`, '.kubernetes.labels.foobar', `.kubernetes.labels."foo-bar/baz"`
-//
-// +kubebuilder:validation:Pattern:=`^(\.[a-zA-Z0-9_]+|\."[^"]+")(\.[a-zA-Z0-9_]+|\."[^"]+")*$`
-type FieldPath string
-
 // FilterSpec defines a filter for log messages.
 //
 // +kubebuilder:validation:XValidation:rule="self.type != 'kubeAPIAudit' || has(self.kubeAPIAudit)", message="Additional type specific spec is required for the filter type"

api/observability/v1/output_types.go

@@ -1045,6 +1045,53 @@ type Splunk struct {
 	// +kubebuilder:validation:Pattern:=`^(([a-zA-Z0-9-_.\/])*(\{(\.[a-zA-Z0-9_]+|\."[^"]+")+((\|\|)(\.[a-zA-Z0-9_]+|\.?"[^"]+")+)*\|\|"[^"]*"\})*)*$`
 	// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Index",xDescriptors={"urn:alm:descriptor:com.tectonic.ui:text"}
 	Index string `json:"index,omitempty"`
+
+	// IndexedFields are the list of fields to be indexed by Splunk, increase storage usage, they should be used sparingly and only for high-value fields that provide significant search benefits.
+	// Nested fields are flattened into top-level fields.
+	// Field paths are joined using dot notation, and unsupported characters are replaced with underscores (_).
+	// Non-string values are automatically converted to strings (e.g., 3 → "3", true → "true").
+	// Object values serialized as JSON strings (e.g., { status: 200 } → "{\"status\":200}").
+	//
+	// Examples: [`.kubernetes`, `.log_type`, '.kubernetes.labels.foobar', `.kubernetes.labels."foo-bar/baz"`]
+	// +nullable
+	// +kubebuilder:validation:Optional
+	// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Indexed Fields"
+	IndexedFields []FieldPath `json:"indexedFields,omitempty"`
+
+	// Source identifies the origin of a log event.
+	// The Source can be a combination of static and dynamic values consisting of field paths followed by `||` followed by another field path or a static value.
+	// A dynamic value is encased in single curly brackets `{}` and MUST end with a static fallback value separated with `||`.
+	// Static values can only contain alphanumeric characters along with dashes, underscores, dots and forward slashes.
+	// If not specified will be detected according to .log_source and .log_type value.
+	// Details see in: docs/features/logforwarding/outputs/splunk-forwarding.adoc
+	//
+	// Example:
+	//
+	//  1. foo-{.bar||"none"}
+	//
+	//  2. {.foo||.bar||"missing"}
+	//
+	//  3. foo.{.bar.baz||.qux.quux.corge||.grault||"nil"}-waldo.fred{.plugh||"none"}
+	//
+	// +kubebuilder:validation:Optional
+	// +kubebuilder:validation:Pattern:=`^(([a-zA-Z0-9-_.\/])*(\{(\.[a-zA-Z0-9_]+|\."[^"]+")+((\|\|)(\.[a-zA-Z0-9_]+|\.?"[^"]+")+)*\|\|"[^"]*"\})*)*$`
+	// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Source",xDescriptors={"urn:alm:descriptor:com.tectonic.ui:text"}
+	Source string `json:"source,omitempty"`
+
+	// PayloadKey specifies record field to use as payload.
+	// The PayloadKey must be a single field path.
+	//
+	// Field paths must only contain alphanumeric and underscores. Any field with other characters must be quoted.
+	//
+	// By default, payloadKey is not set, which means the complete log record is forwarded as the payload.
+	// Use payloadKey carefully. Selecting a single field as the payload may cause other important information in the
+	// log to be dropped, potentially leading to inconsistent or incomplete log events.
+	//
+	// Examples: `.kubernetes`, `.log_type`, '.kubernetes.labels.foobar', `.kubernetes.labels."foo-bar/baz"`
+	//
+	// +kubebuilder:validation:Optional
+	// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Payload Key",xDescriptors={"urn:alm:descriptor:com.tectonic.ui:text"}
+	PayloadKey FieldPath `json:"payloadKey,omitempty"`
 }
 
 // SyslogRFCType sets which RFC the generated messages conform to.

internal/generator/vector/helpers/helpers.go

@@ -5,7 +5,9 @@ import (
 	v1 "github.com/openshift/cluster-logging-operator/api/observability/v1"
 	"github.com/openshift/cluster-logging-operator/internal/constants"
 	"path/filepath"
+	"regexp"
 	"sort"
+	"strconv"
 	"strings"
 	"sync"
 
@@ -14,6 +16,9 @@ import (
 
 const VectorSecretID = "kubernetes_secret"
 
+// Match quoted strings like "foo" or "foo/bar-baz"
+var quoteRegex = regexp.MustCompile(`^".+"$`)
+
 var (
 	Replacer         = strings.NewReplacer(" ", "_", "-", "_", ".", "_")
 	listenAllAddress string
@@ -80,3 +85,71 @@ func SecretFrom(secretKey *v1.SecretReference) string {
 	}
 	return ""
 }
+
+// GenerateQuotedPathSegmentArrayStr generates the final string of the array of array of path segments
+// and array of flattened path with replaced not allowed symbols to feed into VRL
+// E.g
+// [.kubernetes.namespace_labels."bar/baz0-9.test"] -> ([["kubernetes","namespace_labels","bar/baz0-9.test"]], ["_kubernetes_namespace_labels_bar_baz0-9_test"])
+func GenerateQuotedPathSegmentArrayStr(fieldPathArray []v1.FieldPath) (string, string) {
+	var quotedPathArray []string
+	var flattenedArray []string
+
+	for _, fieldPath := range fieldPathArray {
+		pathStr := string(fieldPath)
+
+		if strings.ContainsAny(pathStr, "/.") {
+			flat := strings.NewReplacer(".", "_", "\"", "", "/", "_").Replace(pathStr)
+			flat = strings.TrimPrefix(flat, "_")
+			flattenedArray = append(flattenedArray, strconv.Quote(flat))
+		}
+
+		splitSegments := SplitPath(pathStr)
+		quotedSegments := QuotePathSegments(splitSegments)
+		quotedPathArray = append(quotedPathArray, fmt.Sprintf("[%s]", strings.Join(quotedSegments, ",")))
+	}
+
+	return fmt.Sprintf("[%s]", strings.Join(quotedPathArray, ",")),
+		fmt.Sprintf("[%s]", strings.Join(flattenedArray, ","))
+}
+
+// SplitPath splits a fieldPath by `.` and reassembles the quoted path segments that also contain `.`
+// Example: `.foo."@some"."d.f.g.o111-22/333".foo_bar`
+// Resultant Array: ["foo","@some",`"d.f.g.o111-22/333"`,"foo_bar"]
+func SplitPath(path string) []string {
+	var result []string
+
+	segments := strings.Split(path, ".")
+
+	var currSegment string
+	for _, part := range segments {
+		if part == "" {
+			continue
+		} else if strings.HasPrefix(part, `"`) && strings.HasSuffix(part, `"`) {
+			result = append(result, part)
+		} else if strings.HasPrefix(part, `"`) {
+			currSegment = part
+		} else if strings.HasSuffix(part, `"`) {
+			currSegment += "." + part
+			result = append(result, currSegment)
+			currSegment = ""
+		} else if currSegment != "" {
+			currSegment += "." + part
+		} else {
+			result = append(result, part)
+		}
+	}
+	return result
+}
+
+// QuotePathSegments quotes all path segments as needed for VRL
+func QuotePathSegments(pathArray []string) []string {
+	for i, field := range pathArray {
+		// Don't surround in quotes if already quoted
+		if quoteRegex.MatchString(field) {
+			continue
+		}
+		// Put quotes around path segments
+		pathArray[i] = fmt.Sprintf("%q", field)
+	}
+	return pathArray
+}

internal/generator/vector/helpers/helpers_test.go

@@ -0,0 +1,61 @@
+package helpers
+
+import (
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	obs "github.com/openshift/cluster-logging-operator/api/observability/v1"
+)
+
+var _ = Describe("helpers functions", func() {
+	Context("#GenerateQuotedPathSegmentArrayStr", func() {
+		It("should generate array of path segments and flat path for single value", func() {
+			pathExpression := []obs.FieldPath{`.kubernetes.labels.foo`}
+			expectedArrayPath := `[["kubernetes","labels","foo"]]`
+			expectedFlatPath := `["kubernetes_labels_foo"]`
+			arrayPath, flatPath := GenerateQuotedPathSegmentArrayStr(pathExpression)
+			Expect(arrayPath).To(Equal(expectedArrayPath))
+			Expect(flatPath).To(Equal(expectedFlatPath))
+		})
+		It("should generate array of path segments and flat path for multiple value", func() {
+			pathExpression := []obs.FieldPath{`.kubernetes.labels.foo`, `.kubernetes.labels.bar`}
+			expectedArrayPath := `[["kubernetes","labels","foo"],["kubernetes","labels","bar"]]`
+			expectedFlatPath := `["kubernetes_labels_foo","kubernetes_labels_bar"]`
+			arrayPath, flatPath := GenerateQuotedPathSegmentArrayStr(pathExpression)
+			Expect(arrayPath).To(Equal(expectedArrayPath))
+			Expect(flatPath).To(Equal(expectedFlatPath))
+		})
+		It("should generate array of path segments and escaped flat path", func() {
+			pathExpression := []obs.FieldPath{`.kubernetes.labels."bar/baz0-9.test"`}
+			expectedArrayPath := `[["kubernetes","labels","bar/baz0-9.test"]]`
+			expectedFlatPath := `["kubernetes_labels_bar_baz0-9_test"]`
+			arrayPath, flatPath := GenerateQuotedPathSegmentArrayStr(pathExpression)
+			Expect(arrayPath).To(Equal(expectedArrayPath))
+			Expect(flatPath).To(Equal(expectedFlatPath))
+		})
+		It("should generate array of path segments and escaped flat path", func() {
+			pathExpression := []obs.FieldPath{`.foo.bar."foo.bar.baz-ok".foo123."bar/baz0-9.test"`, `.foo.bar`}
+			expectedArrayPath := `[["foo","bar","foo.bar.baz-ok","foo123","bar/baz0-9.test"],["foo","bar"]]`
+			expectedFlatPath := `["foo_bar_foo_bar_baz-ok_foo123_bar_baz0-9_test","foo_bar"]`
+			arrayPath, flatPath := GenerateQuotedPathSegmentArrayStr(pathExpression)
+			Expect(arrayPath).To(Equal(expectedArrayPath))
+			Expect(flatPath).To(Equal(expectedFlatPath))
+		})
+	})
+
+	DescribeTable("#SplitPath generates correct array of path segments", func(path string, expectedArray []string) {
+		Expect(SplitPath(path)).To(Equal(expectedArray))
+	},
+		Entry("with single segment", `.foo`, []string{"foo"}),
+		Entry("with 2 segments", `.foo.bar`, []string{"foo", "bar"}),
+		Entry("with first segment in quotes", `."@foobar"`, []string{`"@foobar"`}),
+		Entry("with 1 quoted segment and one with quotes", `.foo."bar111-22/333"`, []string{"foo", `"bar111-22/333"`}),
+		Entry("with 2 non quoted segments and one quoted segment ", `.foo.bar."baz111-22/333"`, []string{"foo", "bar", `"baz111-22/333"`}),
+		Entry("with multiple quoted and unquoted segments", `.foo."@some"."d.f.g.o111-22/333".foo_bar`, []string{"foo", `"@some"`, `"d.f.g.o111-22/333"`, "foo_bar"}))
+
+	DescribeTable("#QuotePathSegments generates array with path segments quoted", func(pathSegments []string, expectedArray []string) {
+		Expect(QuotePathSegments(pathSegments)).To(Equal(expectedArray))
+	},
+		Entry("single value", []string{"foo"}, []string{`"foo"`}),
+		Entry("multiple value", []string{"foo", "bar.zip", `"foo-bar"`}, []string{`"foo"`, `"bar.zip"`, `"foo-bar"`}),
+	)
+})