Description
The built-in pfsense parser didn't work for me, it did not match the format of my pfSense netgate plus so I rebuilt it to work with our logging format. Hopefully this will be useful for someone else.
// ************ STANDARD FIELDS ************
| Parser.version := "1.0.0"
| Vendor := "netgate"
| event.module := "pfsense"
| ecs.version := "8.11.0"
| Cps.version := "1.0.0"
// Log format documentation
// https://docs.netgate.com/pfsense/en/latest/monitoring/logs/raw-filter-format.html
| /<(?\d+)>(?\d+)\s(?<@timestamp>\S+)\s(?\S+)\s+(?<Vendor.logtype>\w+)\s+(?<Vendor.pid>\d+)\s+(?<_data>.*)$/
| parseTimestamp("yyyy-MM-dd'T'HH:mm:ss[.SSSSSS]XXX", field="@timestamp")
| log.syslog.priority := Vendor.syslog.priority
| log.syslog.appname := Vendor.logtype
| log.syslog.procid := Vendor.pid
| Vendor.logtype match {
"filterlog" =>
parsecsv(_data, columns=[
_1, _2, _3, _4, _5, _6, _7, _8, _ip_version, _10,
_11, _12, _13, _ipv6_protocol_id, _15, _ipv4_protocol_id,
_17, _ipv6_icmp_type, _19, _20, _ipv4_icmp_type
])
| case {
// IPv4 TCP
_ip_version=4 _ipv4_protocol_id=6
| parsecsv(_data, columns=[
Vendor.rule_number, Vendor.sub_rule_number, Vendor.anchor, Vendor.tracker,
Vendor.real_interface, Vendor.reason, Vendor.action, Vendor.direction,
Vendor.ip_version, Vendor.tos, Vendor.ecn, Vendor.ttl, Vendor.id, Vendor.offset,
Vendor.flags, Vendor.protocol_id, Vendor.protocol, Vendor.length, Vendor.src_ip,
Vendor.dst_ip, Vendor.src_port, Vendor.dst_port, Vendor.data_length,
Vendor.tcp_flags, Vendor.seq_number, Vendor.ack, Vendor.window, Vendor.urg,
Vendor.options
], excludeEmpty=true);
// IPv4 UDP
_ip_version=4 _ipv4_protocol_id=17
| parsecsv(_data, columns=[
Vendor.rule_number, Vendor.sub_rule_number, Vendor.anchor, Vendor.tracker,
Vendor.real_interface, Vendor.reason, Vendor.action, Vendor.direction,
Vendor.ip_version, Vendor.tos, Vendor.ecn, Vendor.ttl, Vendor.id, Vendor.offset,
Vendor.flags, Vendor.protocol_id, Vendor.protocol, Vendor.length, Vendor.src_ip,
Vendor.dst_ip, Vendor.src_port, Vendor.dst_port, Vendor.data_length
], excludeEmpty=true);
// IPv4 ICMP
_ip_version=4 _ipv4_protocol_id=1
| parsecsv(_data, columns=[
Vendor.rule_number, Vendor.sub_rule_number, Vendor.anchor, Vendor.tracker,
Vendor.real_interface, Vendor.reason, Vendor.action, Vendor.direction,
Vendor.ip_version, Vendor.tos, Vendor.ecn, Vendor.ttl, Vendor.id, Vendor.offset,
Vendor.flags, Vendor.protocol_id, Vendor.protocol, Vendor.length, Vendor.src_ip,
Vendor.dst_ip, Vendor.icmp_type
], excludeEmpty=true);
// Remaining IPv4
_ip_version=4
| parsecsv(_data, columns=[
Vendor.rule_number, Vendor.sub_rule_number, Vendor.anchor, Vendor.tracker,
Vendor.real_interface, Vendor.reason, Vendor.action, Vendor.direction,
Vendor.ip_version, Vendor.tos, Vendor.ecn, Vendor.ttl, Vendor.id, Vendor.offset,
Vendor.flags, Vendor.protocol_id, Vendor.protocol, Vendor.length, Vendor.src_ip,
Vendor.dst_ip
], excludeEmpty=true);
// IPv6 TCP
_ip_version=6 _ipv6_protocol_id=6
| parsecsv(_data, columns=[
Vendor.rule_number, Vendor.sub_rule_number, Vendor.anchor, Vendor.tracker,
Vendor.real_interface, Vendor.reason, Vendor.action, Vendor.direction,
Vendor.ip_version, Vendor.class, Vendor.flow_label, Vendor.hop_limit, Vendor.protocol,
Vendor.protocol_id, Vendor.length, Vendor.src_ip, Vendor.dst_ip, Vendor.src_port,
Vendor.dst_port, Vendor.data_length, Vendor.tcp_flags, Vendor.seq_number,
Vendor.ack, Vendor.window, Vendor.urg, Vendor.options
], excludeEmpty=true);
// IPv6 UDP
_ip_version=6 _ipv6_protocol_id=17
| parsecsv(_data, columns=[
Vendor.rule_number, Vendor.sub_rule_number, Vendor.anchor, Vendor.tracker,
Vendor.real_interface, Vendor.reason, Vendor.action, Vendor.direction,
Vendor.ip_version, Vendor.class, Vendor.flow_label, Vendor.hop_limit, Vendor.protocol,
Vendor.protocol_id, Vendor.length, Vendor.src_ip, Vendor.dst_ip, Vendor.src_port,
Vendor.dst_port, Vendor.data_length
], excludeEmpty=true);
// IPv6 ICMP
_ip_version=6 _ipv6_protocol_id=1
| parsecsv(_data, columns=[
Vendor.rule_number, Vendor.sub_rule_number, Vendor.anchor, Vendor.tracker,
Vendor.real_interface, Vendor.reason, Vendor.action, Vendor.direction,
Vendor.ip_version, Vendor.class, Vendor.flow_label, Vendor.hop_limit, Vendor.protocol,
Vendor.protocol_id, Vendor.length, Vendor.src_ip, Vendor.dst_ip, Vendor.icmp_type
], excludeEmpty=true);
// Remaining IPv6
_ip_version=6
| parsecsv(_data, columns=[
Vendor.rule_number, Vendor.sub_rule_number, Vendor.anchor, Vendor.tracker,
Vendor.real_interface, Vendor.reason, Vendor.action, Vendor.direction,
Vendor.ip_version, Vendor.class, Vendor.flow_label, Vendor.hop_limit, Vendor.protocol,
Vendor.protocol_id, Vendor.length, Vendor.src_ip, Vendor.dst_ip
], excludeEmpty=true);
// Anything else
*
}
// Remove the fields used to determine branching
| drop([_ip_version, _ipv6_protocol_id, _ipv4_protocol_id, _ipv6_icmp_type, _ipv4_icmp_type, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, _14, _15, _16, _17, _18, _19, _20])
// ECS mapping for filterlogs
| event.kind := "event"
| event.category[0] := "network"
| event.type[0] := "connection"
| event.dataset := "pfsense.filterlog"
| Vendor.action match {
"pass" =>
event.type[1] := "allowed"
| event.outcome := "success"
;
"block" =>
event.type[1] := "denied"
| event.outcome := "failure"
;
* =>
event.outcome := "unknown"
;
}
| network.transport := lower(Vendor.protocol)
| rule.id := Vendor.rule_number
| event.reason := Vendor.reason
| event.action := Vendor.action
| source.ip := Vendor.src_ip
| source.port := Vendor.src_port
| destination.ip := Vendor.dst_ip
| destination.port := Vendor.dst_port
| case {
test(regex("\.", field=Vendor.real_interface))
| vlan.id := splitString(Vendor.real_interface, by="\.", index=1);
vlan.id := "none"
}
; // End of parsing filter logs
// Let any remaining logs be ingested without further action
* => *
}
| drop(_data)