Fieldref support (#137)

sigma/backends/loki/loki.py

@@ -44,6 +44,7 @@
     SigmaString,
     SigmaNull,
     SigmaNumber,
+    SigmaFieldReference,
 )
 from warnings import warn
 from yaml import dump
@@ -83,6 +84,7 @@ class LogQLDeferredType:
     CIDR = auto()
     REGEXP = auto()
     OR_STR = auto()
+    FIELD_REF = auto()
 
 
 @dataclass
@@ -180,6 +182,35 @@ def finalize_expression(self) -> str:
         return f"{self.op} {or_value}"
 
 
+@dataclass
+class LogQLDeferredFieldRefExpression(DeferredQueryExpression):
+    """'Defer' field reference matching to pipelined command **AFTER** main search expression."""
+
+    field: str
+    value: str
+    field_tracker: int
+
+    def finalize_expression(self) -> str:
+        return f"match_{self.field_tracker}=`{{{{ if eq .{self.field} .{self.value} }}}}true{{{{ else }}}}false{{{{ end }}}}`"
+
+
+@dataclass
+class LogQLDeferredFieldRefFilterExpression(DeferredQueryExpression):
+    """
+    'Defer' field reference matching to after the label_format expressions
+    """
+
+    field_tracker: int
+    op = "true"
+
+    def negate(self) -> DeferredQueryExpression:
+        self.op = "false"
+        return self
+
+    def finalize_expression(self) -> str:
+        return f"match_{self.field_tracker}=`{self.op}`"
+
+
 LogQLLineFilterInfo = NamedTuple(
     "LogQLLineFilterInfo",
     [("value", str), ("negated", bool), ("deftype", auto)],
@@ -299,6 +330,9 @@ def set_expression_templates(negated: bool) -> None:
     add_line_filters: bool = False
     case_sensitive: bool = False
 
+    # Field Ref Match Tracker
+    field_ref_tracker: int = 0
+
     def __init__(
         self,
         processing_pipeline: Optional[ProcessingPipeline] = None,
@@ -881,6 +915,34 @@ def convert_condition_and(
             )
         )
 
+    def convert_condition_field_eq_field(
+        self, cond: SigmaFieldReference, state: ConversionState
+    ) -> Union[str, DeferredQueryExpression]:
+        """
+        Constructs a condition that compares two fields in a log line to enable us to
+        search for logs where the values of two labels are the same.
+        """
+
+        if isinstance(cond, ConditionFieldEqualsValueExpression):
+            if isinstance(cond.value, SigmaFieldReference):
+                field1, field2 = self.convert_condition_field_eq_field_escape_and_quote(
+                    cond.field, cond.value.field
+                )
+                # This gets added by the base class to the state, so we don't need
+                # to return this here, see __post_init__()
+                LogQLDeferredFieldRefExpression(
+                    state, field1, field2, self.field_ref_tracker
+                )
+                expr = LogQLDeferredFieldRefFilterExpression(
+                    state, self.field_ref_tracker
+                )
+                if getattr(cond, "negated", False):
+                    expr.negate()
+                self.field_ref_tracker += 1
+
+                return expr
+        return ""
+
     def convert_condition_field_eq_val(
         self, cond: ConditionFieldEqualsValueExpression, state: ConversionState
     ) -> Union[str, DeferredQueryExpression]:
@@ -1039,12 +1101,45 @@ def finalize_query(
         elif query is None:
             query = ""
         if state.has_deferred():
-            query = self.deferred_separator.join(
-                (
-                    deferred_expression.finalize_expression()
-                    for deferred_expression in state.deferred
+            standard_deferred = [
+                expression.finalize_expression()
+                for expression in state.deferred
+                if not isinstance(
+                    expression,
+                    (
+                        LogQLDeferredFieldRefExpression,
+                        LogQLDeferredFieldRefFilterExpression,
+                    ),
+                )
+            ]
+            field_refs = [
+                expression.finalize_expression()
+                for expression in state.deferred
+                if isinstance(expression, LogQLDeferredFieldRefExpression)
+            ]
+            field_ref_filters = [
+                expression.finalize_expression()
+                for expression in state.deferred
+                if isinstance(expression, LogQLDeferredFieldRefFilterExpression)
+            ]
+            field_ref_expression = ""
+            field_ref_filters_expression = ""
+            if len(field_refs) > 0:
+                label_fmt = ",".join(field_refs)
+                field_ref_expression = (
+                    "| " if len(query) > 0 else f"| {self.select_log_parser(rule)} | "
+                ) + f"label_format {label_fmt}"
+                filter_fmt = " " + self.and_token + " "
+                field_ref_filters_expression = (
+                    f" | {filter_fmt.join(field_ref_filters)}"
                 )
-            ) + (" " + query if len(query) > 0 else "")
+
+            query = (
+                self.deferred_separator.join(standard_deferred)
+                + (" " + query if len(query) > 0 else "")
+                + field_ref_expression
+                + field_ref_filters_expression
+            )
             # Since we've already processed the deferred parts, we can clear them
             state.deferred.clear()
         if rule.fields and len(rule.fields) > 0:

tests/test_backend_loki_fieldref.py

@@ -0,0 +1,164 @@
+import pytest
+from sigma.backends.loki import LogQLBackend
+from sigma.collection import SigmaCollection
+from sigma.processing.pipeline import ProcessingPipeline
+
+
+@pytest.fixture
+def loki_backend():
+    return LogQLBackend(add_line_filters=True)
+
+
+# Testing line filters introduction
+def test_loki_field_ref_single(loki_backend: LogQLBackend):
+    assert (
+        loki_backend.convert(
+            SigmaCollection.from_yaml(
+                """
+            title: Test
+            status: test
+            logsource:
+                category: test_category
+                product: test_product
+            detection:
+                sel:
+                    field|fieldref: fieldA
+                condition: sel
+        """
+            )
+        )
+        == [
+            '{job=~".+"} | logfmt | label_format match_0=`{{ if eq .field .fieldA }}true{{ else }}false{{ end }}` | match_0=`true`'
+        ]
+    )
+
+
+def test_loki_field_ref_multi(loki_backend: LogQLBackend):
+    assert (
+        loki_backend.convert(
+            SigmaCollection.from_yaml(
+                """
+            title: Test
+            status: test
+            logsource:
+                category: test_category
+                product: test_product
+            detection:
+                sel:
+                    field1|fieldref: fieldA
+                    field2|fieldref: fieldB
+                condition: sel
+            """
+            )
+        )
+        == [
+            '{job=~".+"} | logfmt | label_format match_0=`{{ if eq .field1 .fieldA }}true{{ else }}false{{ end }}`,match_1=`{{ if eq .field2 .fieldB }}true{{ else }}false{{ end }}` | match_0=`true` and match_1=`true`'
+        ]
+    )
+
+
+def test_loki_field_ref_json(loki_backend: LogQLBackend):
+    assert (
+        loki_backend.convert(
+            SigmaCollection.from_yaml(
+                """
+            title: Test
+            status: test
+            logsource:
+                category: test_category
+                product: windows 
+            detection:
+                sel:
+                    field|fieldref: fieldA
+                condition: sel
+            """
+            )
+        )
+        == [
+            '{job=~"eventlog|winlog|windows|fluentbit.*"} | json | label_format match_0=`{{ if eq .field .fieldA }}true{{ else }}false{{ end }}` | match_0=`true`'
+        ]
+    )
+
+
+def test_loki_field_ref_json_multi_selection(loki_backend: LogQLBackend):
+    assert (
+        loki_backend.convert(
+            SigmaCollection.from_yaml(
+                """
+            title: Test
+            status: test
+            logsource:
+                category: test_category
+                product: windows 
+            detection:
+                sel:
+                    field1|fieldref: fieldA
+                    field2: Something
+                condition: sel
+            """
+            )
+        )
+        == [
+            '{job=~"eventlog|winlog|windows|fluentbit.*"}  | json | field2=~`(?i)Something`| label_format match_0=`{{ if eq .field1 .fieldA }}true{{ else }}false{{ end }}` | match_0=`true`'
+        ]
+    )
+
+
+def test_loki_field_ref_negated(loki_backend: LogQLBackend):
+    assert (
+        loki_backend.convert(
+            SigmaCollection.from_yaml(
+                """
+            title: Test
+            status: test
+            logsource:
+                category: test_category
+                product: windows
+            detection:
+                sel:
+                    field|fieldref: fieldA
+                sel2:
+                    field2|fieldref: fieldB
+                condition: sel and not sel2
+            """
+            )
+        )
+        == [
+            '{job=~"eventlog|winlog|windows|fluentbit.*"} | json | label_format match_0=`{{ if eq .field .fieldA }}true{{ else }}false{{ end }}`,match_1=`{{ if eq .field2 .fieldB }}true{{ else }}false{{ end }}` | match_0=`true` and match_1=`false`'
+        ]
+    )
+
+
+def test_loki_field_ref_with_pipeline(loki_backend: LogQLBackend):
+    pipeline = ProcessingPipeline.from_yaml(
+        """
+        name: Test Pipeline
+        priority: 20
+        transformations:
+            - id: field_prefix
+              type: field_name_prefix
+              prefix: "event_"
+        """
+    )
+    loki_backend.processing_pipeline = pipeline
+
+    assert (
+        loki_backend.convert(
+            SigmaCollection.from_yaml(
+                """
+            title: Test
+            status: test
+            logsource:
+                category: test_category
+                product: windows
+            detection:
+                sel:
+                    field|fieldref: fieldA
+                condition: sel
+            """
+            )
+        )
+        == [
+            '{job=~"eventlog|winlog|windows|fluentbit.*"} | json | label_format match_0=`{{ if eq .event_field .event_fieldA }}true{{ else }}false{{ end }}` | match_0=`true`'
+        ]
+    )