Skip to content

Commit e5d19bc

Browse files
dikkadevdikkadev
committed
k8s: my current state of work
This commits my current progress in re-creating the docker-compose pipeline for PeekabooAV in Kubernetes. That includes deployments, services, and hard-coded config files for each step in the pipeline, modeled after what was done in the [pipeline](/sett17/peekabooav-installer/tree/pipeline). The yamls for cortex, and the set-up job, are included, although cortex does currently not work inside of Kubernetes, due to the missing docker runner. There is an open [PR](/TheHive-Project/Cortex/pull/349) and corresponding issue. Except for above mentioned cortex, the pipeline is fully functional. Meaning one can send an email to the postfix_tx deployment, which is then sent to the postfix_rx deployment and then processed by rspamd and Peekaboo. This was tested and developed with microk8s and a single node.
1 parent 1b69fff commit e5d19bc

File tree

9 files changed

+767
-0
lines changed

9 files changed

+767
-0
lines changed

‎k8s/cortex_own.yaml‎

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
apiVersion: v1
2+
kind: Service
3+
metadata:
4+
name: cortex
5+
namespace: peekabooav-pipeline
6+
spec:
7+
selector:
8+
app: cortex
9+
sessionAffinity: None
10+
type: NodePort
11+
ports:
12+
- name: cortex
13+
port: 9001
14+
protocol: TCP
15+
targetPort: 9001
16+
---
17+
apiVersion: apps/v1
18+
kind: Deployment
19+
metadata:
20+
name: cortex
21+
namespace: peekabooav-pipeline
22+
labels:
23+
app: cortex
24+
spec:
25+
replicas: 1
26+
selector:
27+
matchLabels:
28+
app: cortex
29+
template:
30+
metadata:
31+
labels:
32+
app: cortex
33+
spec:
34+
containers:
35+
- name: cortex
36+
image: thehiveproject/cortex:3.1.4
37+
volumeMounts:
38+
- name: application
39+
mountPath: /etc/cortex/application.conf
40+
subPath: application.conf
41+
- name: analyzers
42+
mountPath: /etc/cortex/analyzers.json
43+
subPath: analyzers.json
44+
env:
45+
- name: CORTEX_ADMIN_PASSWORD
46+
value: dikka
47+
- name: analyzer_urls
48+
value: /etc/cortex/analyzers.json
49+
ports:
50+
- containerPort: 9001
51+
name: cortex
52+
protocol: TCP
53+
dnsPolicy: ClusterFirst
54+
volumes:
55+
- name: application
56+
configMap:
57+
name: cortex-application-conf
58+
- name: analyzers
59+
configMap:
60+
name: cortex-analyzers-json
61+
---
62+
apiVersion: v1
63+
kind: ConfigMap
64+
metadata:
65+
name: cortex-application-conf
66+
namespace: peekabooav-pipeline
67+
selfLink: /api/v1/namespaces/peekabooav-pipeline/configmaps/cortex-application-conf
68+
data:
69+
application.conf: |
70+
auth.method.basic=true
71+
---
72+
apiVersion: v1
73+
kind: ConfigMap
74+
metadata:
75+
name: cortex-analyzers-json
76+
namespace: peekabooav-pipeline
77+
selfLink: /api/v1/namespaces/peekabooav-pipeline/configmaps/cortex-analyzers-json
78+
data:
79+
analyzers.json: |
80+
[
81+
{
82+
"name": "FileInfo",
83+
"version": "8.0",
84+
"author": "TheHive-Project",
85+
"url": "https://github.com/TheHive-Project/Cortex-Analyzers",
86+
"license": "AGPL-V3",
87+
"description": "Parse files in several formats such as OLE and OpenXML to detect VBA macros, extract their source code, generate useful information on PE, PDF files...",
88+
"dataTypeList": ["file"],
89+
"baseConfig": "FileInfo",
90+
"configurationItems": [
91+
{
92+
"name": "manalyze_enable",
93+
"description": "Wether to enable manalyze submodule or not.",
94+
"type": "boolean",
95+
"required": true,
96+
"multi": false,
97+
"defaultValue": false
98+
},
99+
{
100+
"name": "manalyze_enable_docker",
101+
"description": "Use docker to run Manalyze. Can be used only if not using the docker image of FileInfo",
102+
"type": "boolean",
103+
"required": false,
104+
"multi": false,
105+
"defaultValue": false
106+
},
107+
{
108+
"name": "manalyze_enable_binary",
109+
"description": "Use local binary to run Manalyze. Need to compile it before!",
110+
"type": "boolean",
111+
"required": false,
112+
"multi": false,
113+
"defaultValue": true
114+
},
115+
{
116+
"name": "manalyze_binary_path",
117+
"description": "Path to the Manalyze binary that was compiled before. Keep the default value if using the docker image of FileInfo ",
118+
"type": "string",
119+
"required": false,
120+
"multi": false,
121+
"defaultValue": "/worker/Manalyze/bin/manalyze"
122+
},
123+
{
124+
"name": "floss_enable",
125+
"description": "Enable the use of FireEye FLARE FLOSS",
126+
"type": "boolean",
127+
"required": false,
128+
"multi": false,
129+
"default": false
130+
},
131+
{
132+
"name": "floss_binary_path",
133+
"description": "Path to the FLOSS binary.",
134+
"type": "string",
135+
"required": false,
136+
"multi": false,
137+
"default": "/usr/bin/floss"
138+
},
139+
{
140+
"name": "floss_minimal_string_length",
141+
"description": "Length of strings must be in order to be considered.",
142+
"type": "number",
143+
"required": false,
144+
"multi": false,
145+
"default": 4
146+
}
147+
],
148+
"dockerImage": "cortexneurons/fileinfo:8"
149+
}
150+
]

‎k8s/cortex_setup.yaml‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
# apiVersion: apps/v1
2+
# kind: Deployment
3+
# metadata:
4+
# name: cortex-setup
5+
# namespace: peekabooav-pipeline
6+
# labels:
7+
# app: cortex-setup
8+
# spec:
9+
# replicas: 1
10+
# selector:
11+
# matchLabels:
12+
# app: cortex-setup
13+
# template:
14+
# metadata:
15+
# labels:
16+
# app: cortex-setup
17+
# spec:
18+
# containers:
19+
# - name: cortex-setup
20+
# image: peekabooav_cortex_setup:local
21+
# env:
22+
# - name: ELASTIC_URL
23+
# value: elasticsearch:9200
24+
# - name: CORTEX_URL
25+
# value: cortex:9001
26+
# - name: PEEKABOO_CORTEX_API_TOKEN
27+
# value: dikka
28+
# dnsPolicy: ClusterFirst
29+
apiVersion: batch/v1
30+
kind: Job
31+
metadata:
32+
name: cortex-setup
33+
namespace: peekabooav-pipeline
34+
spec:
35+
template:
36+
spec:
37+
containers:
38+
- name: cortex-setup
39+
image: peekabooav_cortex_setup:local
40+
env:
41+
- name: ELASTIC_URL
42+
value: elasticsearch:9200
43+
- name: CORTEX_URL
44+
value: cortex:9001
45+
- name: PEEKABOO_CORTEX_API_TOKEN
46+
value: dikka
47+
- name: CORTEX_ADMIN_PASSWORD
48+
value: dikka
49+
restartPolicy: Never

‎k8s/elassticsearch.yaml‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
apiVersion: v1
2+
kind: Service
3+
metadata:
4+
name: elasticsearch
5+
namespace: peekabooav-pipeline
6+
spec:
7+
selector:
8+
app: elasticsearch
9+
sessionAffinity: None
10+
type: NodePort
11+
ports:
12+
- name: elasticsearch
13+
port: 9200
14+
protocol: TCP
15+
targetPort: 9200
16+
---
17+
apiVersion: apps/v1
18+
kind: Deployment
19+
metadata:
20+
name: elasticsearch
21+
namespace: peekabooav-pipeline
22+
labels:
23+
app: elasticsearch
24+
spec:
25+
replicas: 1
26+
selector:
27+
matchLabels:
28+
app: elasticsearch
29+
template:
30+
metadata:
31+
labels:
32+
app: elasticsearch
33+
spec:
34+
containers:
35+
- name: elasticsearch
36+
image: elasticsearch:7.16.2
37+
env:
38+
- name: http.host
39+
value: "0.0.0.0"
40+
- name: discovery.type
41+
value: single-node
42+
- name: script.allowed_types
43+
value: inline
44+
- name: thread_pool.search.queue_size
45+
value: "100000"
46+
- name: thread_pool.write.queue_size
47+
value: "10000"
48+
- name: ES_HEAP_SIZE
49+
value: 1g
50+
- name: xpack.security.enabled
51+
value: "false"
52+
- name: cluster.routing.allocation.disk.watermark.flood_stage
53+
value: 99%
54+
- name: TAKE_FILE_OWNERSHIP
55+
value: "1"
56+
dnsPolicy: ClusterFirst

‎k8s/mariadb.yaml‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
apiVersion: v1
2+
kind: Service
3+
metadata:
4+
name: mariadb
5+
namespace: peekabooav-pipeline
6+
spec:
7+
selector:
8+
app: mariadb
9+
sessionAffinity: None
10+
type: NodePort
11+
ports:
12+
- name: mariadb
13+
port: 3306
14+
protocol: TCP
15+
targetPort: 3306
16+
---
17+
apiVersion: apps/v1
18+
kind: Deployment
19+
metadata:
20+
name: mariadb
21+
namespace: peekabooav-pipeline
22+
labels:
23+
app: mariadb
24+
type: database
25+
spec:
26+
replicas: 1
27+
selector:
28+
matchLabels:
29+
app: mariadb
30+
template:
31+
metadata:
32+
labels:
33+
app: mariadb
34+
type: database
35+
spec:
36+
containers:
37+
- name: mariadb
38+
image: mariadb:10.3
39+
readinessProbe:
40+
exec:
41+
command:
42+
- /usr/bin/mysql
43+
- --user=peekaboo
44+
- --password=peekaboo
45+
- --execute
46+
- "SHOW DATABASES;"
47+
initialDelaySeconds: 5
48+
periodSeconds: 5
49+
env:
50+
- name: MARIADB_RANDOM_ROOT_PASSWORD
51+
value: "true"
52+
- name: MARIADB_DATABASE
53+
value: peekaboo
54+
- name: MARIADB_USER
55+
value: peekaboo
56+
- name: MARIADB_PASSWORD
57+
value: peekaboo

0 commit comments

Comments
 (0)