Skip to content

Commit 54d95f3

Browse files
michaelpeacockmichaelpeacock
committed
initial commit
0 parents  commit 54d95f3

33 files changed

+2658
-0
lines changed

‎.gitignore

+68
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
HELP.md
2+
target/
3+
!.mvn/wrapper/maven-wrapper.jar
4+
!**/src/main/**/target/
5+
!**/src/test/**/target/
6+
7+
### STS ###
8+
.apt_generated
9+
.classpath
10+
.factorypath
11+
.project
12+
.settings
13+
.springBeans
14+
.sts4-cache
15+
16+
### IntelliJ IDEA ###
17+
.idea
18+
*.iws
19+
*.iml
20+
*.ipr
21+
22+
### NetBeans ###
23+
/nbproject/private/
24+
/nbbuild/
25+
/dist/
26+
/nbdist/
27+
/.nb-gradle/
28+
build/
29+
!**/src/main/**/build/
30+
!**/src/test/**/build/
31+
32+
### VS Code ###
33+
.vscode/
34+
35+
# Compiled class file
36+
*.class
37+
38+
target
39+
target/*
40+
41+
.DS_Store
42+
43+
.idea
44+
.idea/*
45+
46+
# Maven generated
47+
.flattened-pom.xml
48+
49+
# Log file
50+
*.log
51+
52+
# BlueJ files
53+
*.ctxt
54+
55+
# Mobile Tools for Java (J2ME)
56+
.mtj.tmp/
57+
58+
# Package Files #
59+
*.war
60+
*.nar
61+
*.ear
62+
*.zip
63+
*.tar.gz
64+
*.rar
65+
66+
# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
67+
hs_err_pid*
68+

‎docker-compose.yml

+194
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,194 @@
1+
---
2+
version: '3'
3+
services:
4+
zookeeper:
5+
image: confluentinc/cp-zookeeper:latest
6+
hostname: zookeeper
7+
container_name: zookeeper
8+
ports:
9+
- "2181:2181"
10+
environment:
11+
ZOOKEEPER_CLIENT_PORT: 2181
12+
ZOOKEEPER_TICK_TIME: 2000
13+
14+
broker:
15+
image: confluentinc/cp-server:latest
16+
hostname: broker
17+
container_name: broker
18+
depends_on:
19+
- zookeeper
20+
ports:
21+
- "9092:9092"
22+
- "9101:9101"
23+
healthcheck:
24+
test: nc -z localhost 9092 || exit -1
25+
start_period: 15s
26+
interval: 5s
27+
timeout: 10s
28+
retries: 100
29+
environment:
30+
KAFKA_BROKER_ID: 1
31+
KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
32+
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
33+
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092
34+
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
35+
KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
36+
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
37+
KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR: 1
38+
KAFKA_CONFLUENT_REPORTERS_TELEMETRY_AUTO_ENABLE: 'false'
39+
KAFKA_CONFLUENT_BALANCER_ENABLE: 'false'
40+
KAFKA_CONFLUENT_SCHEMA_REGISTRY_URL: http://schema-registry:8081
41+
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
42+
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
43+
KAFKA_JMX_PORT: 9101
44+
CONFLUENT_SUPPORT_CUSTOMER_ID: 'anonymous'
45+
46+
# schema-registry:
47+
# image: confluentinc/cp-schema-registry:latest
48+
# hostname: schema-registry
49+
# container_name: schema-registry
50+
# depends_on:
51+
# - broker
52+
# ports:
53+
# - "8081:8081"
54+
# environment:
55+
# SCHEMA_REGISTRY_HOST_NAME: schema-registry
56+
# SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'broker:29092'
57+
58+
# connect:
59+
# image: cnfldemos/cp-server-connect-datagen:0.5.3-7.1.0
60+
# hostname: connect
61+
# container_name: connect
62+
# user: root
63+
# depends_on:
64+
# - broker
65+
# - schema-registry
66+
# ports:
67+
# - "8083:8083"
68+
# - "9997:9997"
69+
# - "5140:5140/udp"
70+
# healthcheck:
71+
# interval: 10s
72+
# retries: 20
73+
# test: curl -f http://localhost:8083 || exit 1
74+
# environment:
75+
# CONNECT_BOOTSTRAP_SERVERS: 'broker:29092'
76+
# CONNECT_REST_ADVERTISED_HOST_NAME: connect
77+
# CONNECT_REST_PORT: 8083
78+
# CONNECT_GROUP_ID: compose-connect-group
79+
# CONNECT_CONFIG_STORAGE_TOPIC: _docker-connect-configs
80+
# CONNECT_OFFSET_STORAGE_TOPIC: _docker-connect-offsets
81+
# CONNECT_STATUS_STORAGE_TOPIC: _docker-connect-status
82+
# CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
83+
# CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
84+
# CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
85+
# CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000
86+
# CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter
87+
# CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
88+
# CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8081
89+
# CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
90+
# CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
91+
# CONNECT_PLUGIN_PATH: "/usr/share/java,/usr/share/confluent-hub-components"
92+
# CONNECT_LOG4J_LOGGERS: org.apache.zookeeper=ERROR,org.I0Itec.zkclient=ERROR,org.reflections=ERROR
93+
94+
control-center:
95+
image: confluentinc/cp-enterprise-control-center:latest
96+
hostname: control-center
97+
container_name: control-center
98+
user: root
99+
depends_on:
100+
- broker
101+
# - schema-registry
102+
# - connect
103+
# - ksqldb-server
104+
ports:
105+
- "9021:9021"
106+
environment:
107+
CONTROL_CENTER_BOOTSTRAP_SERVERS: 'broker:29092'
108+
CONTROL_CENTER_CONNECT_CONNECT-DEFAULT_CLUSTER: 'connect:8083'
109+
# The control center server connects to ksqlDB through the docker network
110+
CONTROL_CENTER_KSQL_KSQLDB1_URL: "http://ksqldb-server:8088"
111+
CONTROL_CENTER_KSQL_KSQLDB1_ADVERTISED_URL: https://localhost:8088
112+
CONTROL_CENTER_SCHEMA_REGISTRY_URL: "http://schema-registry:8081"
113+
CONTROL_CENTER_REPLICATION_FACTOR: 1
114+
CONTROL_CENTER_INTERNAL_TOPICS_PARTITIONS: 1
115+
CONTROL_CENTER_MODE_ENABLE: "management"
116+
117+
# ksqldb-server:
118+
# image: confluentinc/cp-ksqldb-server:latest
119+
# hostname: ksqldb-server
120+
# container_name: ksqldb-server
121+
# depends_on:
122+
# - broker
123+
# - connect
124+
# ports:
125+
# - "8088:8088"
126+
# volumes:
127+
# - ./ksqlDB/ksql-extension:/etc/ksql-extension/
128+
# environment:
129+
# KSQL_CONFIG_DIR: "/etc/ksql"
130+
# KSQL_KSQL_EXTENSION_DIR: "/etc/ksql-extension"
131+
# KSQL_BOOTSTRAP_SERVERS: "broker:29092"
132+
# KSQL_HOST_NAME: ksqldb-server
133+
# KSQL_LISTENERS: "http://0.0.0.0:8088"
134+
# KSQL_CACHE_MAX_BYTES_BUFFERING: 0
135+
# KSQL_KSQL_SCHEMA_REGISTRY_URL: "http://schema-registry:8081"
136+
# KSQL_KSQL_CONNECT_URL: "http://connect:8083"
137+
# KSQL_KSQL_HIDDEN_TOPICS: '^_.*'
138+
# KSQL_KSQL_LOGGING_PROCESSING_STREAM_AUTO_CREATE: "true"
139+
# KSQL_KSQL_LOGGING_PROCESSING_TOPIC_AUTO_CREATE: "true"
140+
141+
# confluent-sigma-streams:
142+
# image: confluentinc/confluent-sigma:1.3.2
143+
# container_name: confluent-sigma-streams
144+
# depends_on:
145+
# broker:
146+
# condition: service_healthy
147+
# hostname: confluent-sigma-streams
148+
# environment:
149+
# application_id: 'dns-streams-app'
150+
# bootstrap_server: 'broker:29092'
151+
# schema_registry: 'http://schema-registry:8081'
152+
# data_topic: 'dns'
153+
# output_topic: 'dns-detection'
154+
# sigma_rules_topic: 'sigma-rules'
155+
# sigma_rule_filter_product: 'zeek'
156+
# sigma_rule_filter_service: 'dns'
157+
# sigma_rule_first_match: 'false'
158+
159+
# confluent-sigma-ui:
160+
# image: confluentinc/confluent-sigma-ui:1.3.2
161+
# container_name: confluent-sigma-ui
162+
# depends_on:
163+
# broker:
164+
# condition: service_healthy
165+
# hostname: confluent-sigma-ui
166+
# ports:
167+
# - 8080:8080
168+
# environment:
169+
# bootstrap_server: 'broker:29092'
170+
# schema_registry: 'http://schema-registry:8081'
171+
# sigma_rules_topic: 'sigma-rules'
172+
# confluent_regex_applicationID: 'regex-application'
173+
# confluent_regex_inputTopic: 'splunk-s2s-events'
174+
# confluent_regex_ruleTopic: 'regex-rules'
175+
# confluent_regex_filterField: 'sourcetype'
176+
# confluent_regex_regexField: 'event'
177+
178+
# dns-load-data:
179+
# image: edenhill/kcat:1.7.1
180+
# container_name: dns-load-data
181+
# hostname: dns-load-data
182+
# depends_on:
183+
# broker:
184+
# condition: service_healthy
185+
# volumes:
186+
# - ./demo/data:/tmp/data
187+
# command:
188+
# - -b
189+
# - broker:29092
190+
# - -t
191+
# - dns
192+
# - -P
193+
# - -l
194+
# - /tmp/data/dns.txt

‎kstreamrouter/kstreamsrouter/config/application.yaml

+44
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
spring.cloud.stream:
2+
function:
3+
definition: dynamicRouteSyslogMessages
4+
bindings:
5+
dynamicRouteSyslogMessages-in-0: kstreams.test.input
6+
7+
kafka:
8+
streams:
9+
binder:
10+
applicationId: ks-rcc-router-app
11+
deserializationExceptionHandler: logAndContinue
12+
configuration:
13+
spring.json.trusted.packages: "*"
14+
commit.interval.ms: 2000
15+
state.dir: state-store
16+
default:
17+
security:
18+
protocol: SSL
19+
ssl:
20+
truststore:
21+
location: c:/tmp/kstreams/confluent-dev/kafkaclient-truststore.jks
22+
password: mystorepassword
23+
type: JKS
24+
keystore:
25+
location: c:/tmp/kstreams/confluent-dev/kafkaclient-keystore.jks
26+
password: mystorepassword
27+
brokers: localhost:9092
28+
29+
logging:
30+
level:
31+
#org.springframework: debug
32+
mil:
33+
army:
34+
rcc:
35+
kstreamrouter: DEBUG
36+
37+
38+
properties:
39+
sasl:
40+
jaas:
41+
config: org.apache.kafka.common.security.scram.ScramLoginModule required username=<scram user> password=<scram password>;
42+
mechanism: SCRAM-SHA-512
43+
security:
44+
protocol: SASL_SSL

‎kstreamrouter/kstreamsrouter/config/routingconfig.yaml

+31
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
custom:
2+
customFields:
3+
-
4+
name: "rccName"
5+
value: "RCC-E"
6+
-
7+
name: "siteName"
8+
value: "Kaiserslautern"
9+
10+
routing:
11+
default:
12+
topic: rcc.log.syslog.default
13+
rules:
14+
-
15+
regEx: ".*,TRAFFIC,\\w+.*"
16+
outputTopic: "rcc.log.route1"
17+
-
18+
regEx: ".*\\S+\\s\\d{10}\\s.+?\\d{10}\\s\\d{5}\\s(?:FATAL|NOTICE|DEBUG|INFO|WARN(?:ING)?|ERR(?:OR)?|WARN|CRITICAL).*"
19+
outputTopic: "rcc.log.ciscoise.out"
20+
-
21+
regEx: ".*(?:Local\\sgateway.+Remote\\sgateway|OSPF\\sneighbor.+realm|IKE negotiation|VPN\\s\\S+\\sfrom.+is\\s(?:up|down)|DPD\\sdetected\\speer).*"
22+
outputTopic: "rcc.log.juniper.out"
23+
-
24+
regEx: ".*%ASA.*"
25+
outputTopic: "rcc.log.ciscoasa.out"
26+
-
27+
regEx: ".*(?:f5_asm|Cannot\\sload\\suser\\scredentials|f5_afm|f5_irule|Hostname=.+,SlotId=.+errdefs_msgno|Pool.+now has|SSL Handshake failed for (?:TCP|UDP)|SNMP Trap:.*has become|SNMP_TRAP|Connection error:.*alert\\(\\d+\\)|icrd_child|AUDIT\\s\\-\\s|Virtual Address.*status|No members available for pool|Pool.*member.*status|DoS Auto Ratelimit Threshold|Sync of device group).*"
28+
outputTopic: "rcc.log.f5asm.out"
29+
-
30+
regEx: ".*(?:%FMANFP|%SESSION_MGR|%CLIENT_ORCH|%DOT1X|%APMGR|%CLIENT_EXCLUSION|%SEC-[0-9]-IPACCESSLOGS|%SNMP-[0-9]-AUTHFAIL|%SYS-[0-9]-CONFIG_I).*"
31+
outputTopic: "rcc.log.ciscoiso.out"

0 commit comments

Comments
 (0)