@@ -10,6 +10,9 @@ import (
10
10
11
11
"gopkg.in/yaml.v3"
12
12
"k8s.io/client-go/kubernetes/scheme"
13
+
14
+ "github.com/np-guard/cluster-topology-analyzer/pkg/analyzer"
15
+ "github.com/np-guard/cluster-topology-analyzer/pkg/common"
13
16
)
14
17
15
18
// K8s resources that are relevant for connectivity analysis
@@ -18,8 +21,8 @@ const (
18
21
replicaSet string = "ReplicaSet"
19
22
replicationController string = "ReplicationController"
20
23
deployment string = "Deployment"
21
- statefulset string = "StatefulSet"
22
- daemonset string = "DaemonSet"
24
+ statefulSet string = "StatefulSet"
25
+ daemonSet string = "DaemonSet"
23
26
job string = "Job"
24
27
cronJob string = "CronTab"
25
28
service string = "Service"
@@ -28,57 +31,48 @@ const (
28
31
29
32
var (
30
33
acceptedK8sTypesRegex = fmt .Sprintf ("(^%s$|^%s$|^%s$|^%s$|^%s$|^%s$|^%s$|^%s$|^%s$|^%s$)" ,
31
- pod , replicaSet , replicationController , deployment , daemonset , statefulset , job , cronJob , service , configmap )
32
- acceptedK8sTypes = regexp .MustCompile (acceptedK8sTypesRegex )
33
- yamlSuffix = regexp .MustCompile (".ya?ml$" )
34
+ pod , replicaSet , replicationController , deployment , daemonSet , statefulSet , job , cronJob , service , configmap )
35
+ acceptedK8sTypes = regexp .MustCompile (acceptedK8sTypesRegex )
36
+ yamlSuffix = regexp .MustCompile (".ya?ml$" )
37
+ k8sResourceDecoder = scheme .Codecs .UniversalDeserializer ()
34
38
)
35
39
36
- // rawResourcesInFile represents a single YAML file with multiple K8s resources
37
- type rawResourcesInFile struct {
38
- ManifestFilepath string
39
- rawK8sResources []rawK8sResource
40
- }
41
-
42
- // rawK8sResource stores a raw K8s resource and its kind for later parsing
43
- type rawK8sResource struct {
44
- GroupKind string
45
- RuntimeObject []byte
46
- }
47
-
48
- // resourceFinder is used to locate all relevant K8s resources in a given file-system directory
40
+ // resourceFinder is used to locate all relevant K8s resources in given file-system directories
41
+ // and to convert them into the internal structs, used for later processing.
49
42
type resourceFinder struct {
50
43
logger Logger
51
44
stopOn1stErr bool
52
45
walkFn WalkFunction // for customizing directory scan
46
+
47
+ workloads []common.Resource // accumulates all workload resources found
48
+ services []common.Service // accumulates all service resources found
49
+ configmaps []common.CfgMap // accumulates all ConfigMap resources found
53
50
}
54
51
55
52
// getRelevantK8sResources is the main function of resourceFinder.
56
53
// It scans a given directory using walkFn, looking for all yaml files. It then breaks each yaml into its documents
57
54
// and extracts all K8s resources that are relevant for connectivity analysis.
58
- // The result is stored in a slice of rawResourcesInFile (one per yaml file), each containing a slice of rawK8sResource
59
- func (rf * resourceFinder ) getRelevantK8sResources (repoDir string ) ([] rawResourcesInFile , [] FileProcessingError ) {
55
+ // The resources are stored in the struct, separated to workloads, services and configmaps
56
+ func (rf * resourceFinder ) getRelevantK8sResources (repoDir string ) [] FileProcessingError {
60
57
manifestFiles , fileScanErrors := rf .searchForManifests (repoDir )
61
58
if stopProcessing (rf .stopOn1stErr , fileScanErrors ) {
62
- return nil , fileScanErrors
59
+ return fileScanErrors
63
60
}
64
61
if len (manifestFiles ) == 0 {
65
62
fileScanErrors = appendAndLogNewError (fileScanErrors , noYamlsFound (), rf .logger )
66
- return nil , fileScanErrors
63
+ return fileScanErrors
67
64
}
68
65
69
- parsedObjs := []rawResourcesInFile {}
70
66
for _ , mfp := range manifestFiles {
71
- rawK8sResources , err := rf .parseK8sYaml (mfp )
72
- fileScanErrors = append (fileScanErrors , err ... )
67
+ relMfp := pathWithoutBaseDir (mfp , repoDir )
68
+ errs := rf .parseK8sYaml (mfp , relMfp )
69
+ fileScanErrors = append (fileScanErrors , errs ... )
73
70
if stopProcessing (rf .stopOn1stErr , fileScanErrors ) {
74
- return nil , fileScanErrors
75
- }
76
- if len (rawK8sResources ) > 0 {
77
- manifestFilePath := pathWithoutBaseDir (mfp , repoDir )
78
- parsedObjs = append (parsedObjs , rawResourcesInFile {rawK8sResources : rawK8sResources , ManifestFilepath : manifestFilePath })
71
+ return fileScanErrors
79
72
}
80
73
}
81
- return parsedObjs , fileScanErrors
74
+
75
+ return fileScanErrors
82
76
}
83
77
84
78
// searchForManifests returns a list of YAML files under a given directory (recursively)
@@ -105,14 +99,14 @@ func (rf *resourceFinder) searchForManifests(repoDir string) ([]string, []FilePr
105
99
}
106
100
107
101
// splitByYamlDocuments takes a YAML file and returns a slice containing its documents as raw text
108
- func (rf * resourceFinder ) splitByYamlDocuments (mfp string ) ([]string , []FileProcessingError ) {
102
+ func (rf * resourceFinder ) splitByYamlDocuments (mfp string ) ([][] byte , []FileProcessingError ) {
109
103
fileBuf , err := os .ReadFile (mfp )
110
104
if err != nil {
111
- return [] string {} , appendAndLogNewError (nil , failedReadingFile (mfp , err ), rf .logger )
105
+ return nil , appendAndLogNewError (nil , failedReadingFile (mfp , err ), rf .logger )
112
106
}
113
107
114
108
decoder := yaml .NewDecoder (bytes .NewBuffer (fileBuf ))
115
- documents := []string {}
109
+ documents := [][] byte {}
116
110
documentID := 0
117
111
for {
118
112
var doc yaml.Node
@@ -127,39 +121,67 @@ func (rf *resourceFinder) splitByYamlDocuments(mfp string) ([]string, []FileProc
127
121
if err != nil {
128
122
return documents , appendAndLogNewError (nil , malformedYamlDoc (mfp , doc .Line , documentID , err ), rf .logger )
129
123
}
130
- documents = append (documents , string ( out ) )
124
+ documents = append (documents , out )
131
125
}
132
126
documentID += 1
133
127
}
134
128
return documents , nil
135
129
}
136
130
137
- // parseK8sYaml takes a YAML document and checks if it stands for a relevant K8s resource.
138
- // If yes, it puts it into a rawK8sResource and appends it to the result.
139
- func (rf * resourceFinder ) parseK8sYaml (mfp string ) ([]rawK8sResource , []FileProcessingError ) {
140
- dObjs := []rawK8sResource {}
141
- sepYamlFiles , fileProcessingErrors := rf .splitByYamlDocuments (mfp )
131
+ // parseK8sYaml takes a YAML file and attempts to parse each of its documents into
132
+ // one of the relevant k8s resources
133
+ func (rf * resourceFinder ) parseK8sYaml (mfp , relMfp string ) []FileProcessingError {
134
+ yamlDocs , fileProcessingErrors := rf .splitByYamlDocuments (mfp )
142
135
if stopProcessing (rf .stopOn1stErr , fileProcessingErrors ) {
143
- return nil , fileProcessingErrors
136
+ return fileProcessingErrors
144
137
}
145
138
146
- for docID , doc := range sepYamlFiles {
147
- decode := scheme .Codecs .UniversalDeserializer ().Decode
148
- _ , groupVersionKind , err := decode ([]byte (doc ), nil , nil )
139
+ for docID , doc := range yamlDocs {
140
+ _ , groupVersionKind , err := k8sResourceDecoder .Decode (doc , nil , nil )
149
141
if err != nil {
150
- fileProcessingErrors = appendAndLogNewError (fileProcessingErrors , notK8sResource (mfp , docID , err ), rf .logger )
142
+ fileProcessingErrors = appendAndLogNewError (fileProcessingErrors , notK8sResource (relMfp , docID , err ), rf .logger )
151
143
continue
152
144
}
153
145
if ! acceptedK8sTypes .MatchString (groupVersionKind .Kind ) {
154
- rf .logger .Infof ("in file: %s, document: %d, skipping object with type: %s" , mfp , docID , groupVersionKind .Kind )
146
+ rf .logger .Infof ("in file: %s, document: %d, skipping object with type: %s" , relMfp , docID , groupVersionKind .Kind )
155
147
} else {
156
- d := rawK8sResource {}
157
- d .GroupKind = groupVersionKind .Kind
158
- d .RuntimeObject = []byte (doc )
159
- dObjs = append (dObjs , d )
148
+ kind := groupVersionKind .Kind
149
+ err = rf .parseResource (kind , doc , relMfp )
150
+ if err != nil {
151
+ fileProcessingErrors = appendAndLogNewError (fileProcessingErrors , failedScanningResource (kind , relMfp , err ), rf .logger )
152
+ }
153
+ }
154
+ }
155
+ return fileProcessingErrors
156
+ }
157
+
158
+ // parseResource takes a yaml document, parses it into a K8s resource and puts it into one of the 3 struct slices:
159
+ // the workload resource slice, the Service resource slice, and the ConfigMaps resource slice
160
+ func (rf * resourceFinder ) parseResource (kind string , yamlDoc []byte , manifestFilePath string ) error {
161
+ switch kind {
162
+ case service :
163
+ res , err := analyzer .ScanK8sServiceObject (kind , yamlDoc )
164
+ if err != nil {
165
+ return err
160
166
}
167
+ res .Resource .FilePath = manifestFilePath
168
+ rf .services = append (rf .services , res )
169
+ case configmap :
170
+ res , err := analyzer .ScanK8sConfigmapObject (kind , yamlDoc )
171
+ if err != nil {
172
+ return err
173
+ }
174
+ rf .configmaps = append (rf .configmaps , res )
175
+ default :
176
+ res , err := analyzer .ScanK8sWorkloadObject (kind , yamlDoc )
177
+ if err != nil {
178
+ return err
179
+ }
180
+ res .Resource .FilePath = manifestFilePath
181
+ rf .workloads = append (rf .workloads , res )
161
182
}
162
- return dObjs , fileProcessingErrors
183
+
184
+ return nil
163
185
}
164
186
165
187
// returns a file path without its prefix base dir
@@ -174,3 +196,49 @@ func pathWithoutBaseDir(path, baseDir string) string {
174
196
}
175
197
return relPath
176
198
}
199
+
200
+ // inlineConfigMapRefsAsEnvs appends to the Envs of each given resource the ConfigMap values it is referring to
201
+ // It should only be called after ALL calls to getRelevantK8sResources successfully returned
202
+ func (rf * resourceFinder ) inlineConfigMapRefsAsEnvs () []FileProcessingError {
203
+ cfgMapsByName := map [string ]* common.CfgMap {}
204
+ for cm := range rf .configmaps {
205
+ cfgMapsByName [rf .configmaps [cm ].FullName ] = & rf .configmaps [cm ]
206
+ }
207
+
208
+ parseErrors := []FileProcessingError {}
209
+ for idx := range rf .workloads {
210
+ res := & rf .workloads [idx ]
211
+
212
+ // inline the envFrom field in PodSpec->containers
213
+ for _ , cfgMapRef := range res .Resource .ConfigMapRefs {
214
+ configmapFullName := res .Resource .Namespace + "/" + cfgMapRef
215
+ if cfgMap , ok := cfgMapsByName [configmapFullName ]; ok {
216
+ for _ , v := range cfgMap .Data {
217
+ if analyzer .IsNetworkAddressValue (v ) {
218
+ res .Resource .NetworkAddrs = append (res .Resource .NetworkAddrs , v )
219
+ }
220
+ }
221
+ } else {
222
+ parseErrors = appendAndLogNewError (parseErrors , configMapNotFound (configmapFullName , res .Resource .Name ), rf .logger )
223
+ }
224
+ }
225
+
226
+ // inline PodSpec->container->env->valueFrom->configMapKeyRef
227
+ for _ , cfgMapKeyRef := range res .Resource .ConfigMapKeyRefs {
228
+ configmapFullName := res .Resource .Namespace + "/" + cfgMapKeyRef .Name
229
+ if cfgMap , ok := cfgMapsByName [configmapFullName ]; ok {
230
+ if val , ok := cfgMap .Data [cfgMapKeyRef .Key ]; ok {
231
+ if analyzer .IsNetworkAddressValue (val ) {
232
+ res .Resource .NetworkAddrs = append (res .Resource .NetworkAddrs , val )
233
+ }
234
+ } else {
235
+ err := configMapKeyNotFound (cfgMapKeyRef .Name , cfgMapKeyRef .Key , res .Resource .Name )
236
+ parseErrors = appendAndLogNewError (parseErrors , err , rf .logger )
237
+ }
238
+ } else {
239
+ parseErrors = appendAndLogNewError (parseErrors , configMapNotFound (configmapFullName , res .Resource .Name ), rf .logger )
240
+ }
241
+ }
242
+ }
243
+ return parseErrors
244
+ }
0 commit comments