Skip to content

Commit 242a570

Browse files
authored
Separate dir scanning class (#329)
* rename resourceFinder to resourceAccumulator * Move manifest finding code to a new class * A common function to parse multiple Info objects * rf -> ra * A function to parse multiple YAML files * A function to scan multiple dirs for YAMLs * Get rid of getRelevantK8sResources + update tests * resourceAccumulator no longer needs a walk function * missing check to stop on first error Signed-off-by: ZIV NEVO <[email protected]>
1 parent d59bd1b commit 242a570

6 files changed

+308
-249
lines changed

pkg/analyzer/manifest_finder.go

+65
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
/*
2+
Copyright 2020- IBM Inc. All Rights Reserved.
3+
4+
SPDX-License-Identifier: Apache-2.0
5+
*/
6+
7+
package analyzer
8+
9+
import (
10+
"os"
11+
"path/filepath"
12+
"regexp"
13+
)
14+
15+
var yamlSuffix = regexp.MustCompile(".ya?ml$")
16+
17+
// manifestFinder is a utility class for searching for YAML files
18+
type manifestFinder struct {
19+
logger Logger
20+
stopOn1stErr bool
21+
walkFn WalkFunction // for customizing directory scan
22+
}
23+
24+
// searchForManifestsInDirs is a convenience function to call searchForManifestsInDir() for each path in a slice of dir paths
25+
func (mf *manifestFinder) searchForManifestsInDirs(dirPaths []string) ([]string, []FileProcessingError) {
26+
manifestFiles := []string{}
27+
fileErrors := []FileProcessingError{}
28+
for _, dirPath := range dirPaths {
29+
manifests, errs := mf.searchForManifestsInDir(dirPath)
30+
manifestFiles = append(manifestFiles, manifests...)
31+
fileErrors = append(fileErrors, errs...)
32+
if stopProcessing(mf.stopOn1stErr, errs) {
33+
return nil, fileErrors
34+
}
35+
36+
if len(manifestFiles) == 0 {
37+
fileErrors = appendAndLogNewError(fileErrors, noYamlsFound(), mf.logger)
38+
}
39+
}
40+
return manifestFiles, fileErrors
41+
}
42+
43+
// searchForManifestsInDir returns a list of YAML files under a given directory.
44+
// Directory is scanned using the configured walk function
45+
func (mf *manifestFinder) searchForManifestsInDir(repoDir string) ([]string, []FileProcessingError) {
46+
yamls := []string{}
47+
errors := []FileProcessingError{}
48+
err := mf.walkFn(repoDir, func(path string, f os.DirEntry, err error) error {
49+
if err != nil {
50+
errors = appendAndLogNewError(errors, failedAccessingDir(path, err, path != repoDir), mf.logger)
51+
if stopProcessing(mf.stopOn1stErr, errors) {
52+
return err
53+
}
54+
return filepath.SkipDir
55+
}
56+
if f != nil && !f.IsDir() && yamlSuffix.MatchString(f.Name()) {
57+
yamls = append(yamls, path)
58+
}
59+
return nil
60+
})
61+
if err != nil {
62+
mf.logger.Errorf(err, "Error walking directory: %v", err)
63+
}
64+
return yamls, errors
65+
}

pkg/analyzer/manifest_finder_test.go

+76
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
/*
2+
Copyright 2020- IBM Inc. All Rights Reserved.
3+
4+
SPDX-License-Identifier: Apache-2.0
5+
*/
6+
7+
package analyzer
8+
9+
import (
10+
"errors"
11+
"io/fs"
12+
"os"
13+
"path/filepath"
14+
"testing"
15+
16+
"github.com/stretchr/testify/require"
17+
)
18+
19+
func TestSearchForManifests(t *testing.T) {
20+
dirPath := filepath.Join(getTestsDir(), "bad_yamls")
21+
manFinder := manifestFinder{NewDefaultLogger(), false, filepath.WalkDir}
22+
yamlFiles, errs := manFinder.searchForManifestsInDir(dirPath)
23+
require.Empty(t, errs)
24+
require.Len(t, yamlFiles, 5)
25+
}
26+
27+
func nonRecursiveWalk(root string, fn fs.WalkDirFunc) error {
28+
err := filepath.WalkDir(root, func(path string, f os.DirEntry, err error) error {
29+
if err != nil {
30+
return filepath.SkipDir
31+
}
32+
if f == nil || path != root && f.IsDir() {
33+
return filepath.SkipDir
34+
}
35+
return fn(path, f, err)
36+
})
37+
return err
38+
}
39+
40+
func TestSearchForManifestsNonRecursiveWalk(t *testing.T) {
41+
dirPath := filepath.Join(getTestsDir(), "bad_yamls")
42+
manFinder := manifestFinder{NewDefaultLogger(), false, nonRecursiveWalk}
43+
yamlFiles, errs := manFinder.searchForManifestsInDir(dirPath)
44+
require.Empty(t, errs)
45+
require.Len(t, yamlFiles, 4)
46+
}
47+
48+
func TestSearchForManifestsMultipleDirs(t *testing.T) {
49+
dirPath1 := filepath.Join(getTestsDir(), "k8s_wordpress_example")
50+
dirPath2 := filepath.Join(getTestsDir(), "onlineboutique")
51+
manFinder := manifestFinder{NewDefaultLogger(), false, filepath.WalkDir}
52+
yamlFiles, errs := manFinder.searchForManifestsInDirs([]string{dirPath1, dirPath2})
53+
require.Empty(t, errs)
54+
require.Len(t, yamlFiles, 5)
55+
}
56+
57+
func TestSearchForManifestsMultipleDirsWithErrors(t *testing.T) {
58+
dirPath1 := filepath.Join(getTestsDir(), "k8s_wordpress_example")
59+
dirPath2 := filepath.Join(getTestsDir(), "badPath")
60+
manFinder := manifestFinder{NewDefaultLogger(), false, filepath.WalkDir}
61+
yamlFiles, errs := manFinder.searchForManifestsInDirs([]string{dirPath1, dirPath2})
62+
badDir := &FailedAccessingDirError{}
63+
require.NotEmpty(t, errs)
64+
require.True(t, errors.As(errs[0].Error(), &badDir))
65+
require.Empty(t, yamlFiles)
66+
}
67+
68+
func TestNoYamlsInDir(t *testing.T) {
69+
dirPath := filepath.Join(getTestsDir(), "bad_yamls", "subdir2")
70+
manFinder := manifestFinder{NewDefaultLogger(), false, filepath.WalkDir}
71+
yamlFiles, errs := manFinder.searchForManifestsInDirs([]string{dirPath})
72+
require.Len(t, errs, 1)
73+
noYamls := &NoYamlsFoundError{}
74+
require.True(t, errors.As(errs[0].Error(), &noYamls))
75+
require.Empty(t, yamlFiles)
76+
}

pkg/analyzer/policies_synthesizer.go

+30-30
Original file line numberDiff line numberDiff line change
@@ -167,58 +167,58 @@ func (ps *PoliciesSynthesizer) ConnectionsFromFolderPaths(dirPaths []string) ([]
167167

168168
func (ps *PoliciesSynthesizer) extractConnectionsFromInfos(infos []*resource.Info) (
169169
[]*Resource, []*Connections, []FileProcessingError) {
170-
resFinder := newResourceFinder(ps.logger, ps.stopOnError, ps.walkFn)
171-
fileErrors := []FileProcessingError{}
172-
for _, info := range infos {
173-
err := resFinder.parseInfo(info)
174-
if err != nil {
175-
kind := "<unknown>"
176-
if info != nil && info.Object != nil {
177-
kind = info.Object.GetObjectKind().GroupVersionKind().Kind
178-
}
179-
fileErrors = appendAndLogNewError(fileErrors, failedScanningResource(kind, info.Source, err), ps.logger)
180-
}
170+
resAcc := newResourceAccumulator(ps.logger, ps.stopOnError)
171+
parseErrors := resAcc.parseInfos(infos)
172+
if stopProcessing(ps.stopOnError, parseErrors) {
173+
return nil, nil, parseErrors
181174
}
182175

183-
wls, conns, errs := ps.extractConnections(resFinder)
184-
fileErrors = append(fileErrors, errs...)
185-
return wls, conns, fileErrors
176+
wls, conns, errs := ps.extractConnections(resAcc)
177+
errs = append(parseErrors, errs...)
178+
return wls, conns, errs
186179
}
187180

188-
// Scans the given directory for YAMLs with k8s resources and extracts required connections between workloads
181+
// Scans the given directories for YAMLs with k8s resources and extracts required connections between workloads
189182
func (ps *PoliciesSynthesizer) extractConnectionsFromFolderPaths(dirPaths []string) (
190183
[]*Resource, []*Connections, []FileProcessingError) {
191-
resFinder := newResourceFinder(ps.logger, ps.stopOnError, ps.walkFn)
192-
fileErrors := []FileProcessingError{}
193-
for _, dirPath := range dirPaths {
194-
errs := resFinder.getRelevantK8sResources(dirPath)
195-
fileErrors = append(fileErrors, errs...)
196-
if stopProcessing(ps.stopOnError, errs) {
197-
return nil, nil, fileErrors
198-
}
184+
// Find all manifest YAML files
185+
mf := manifestFinder{ps.logger, ps.stopOnError, ps.walkFn}
186+
manifestFiles, fileErrors := mf.searchForManifestsInDirs(dirPaths)
187+
if stopProcessing(ps.stopOnError, fileErrors) {
188+
return nil, nil, fileErrors
199189
}
200-
wls, conns, errs := ps.extractConnections(resFinder)
190+
191+
// Parse YAMLs and extract relevant resources
192+
resAcc := newResourceAccumulator(ps.logger, ps.stopOnError)
193+
parseErrors := resAcc.parseK8sYamls(manifestFiles)
194+
fileErrors = append(fileErrors, parseErrors...)
195+
if stopProcessing(ps.stopOnError, fileErrors) {
196+
return nil, nil, fileErrors
197+
}
198+
199+
// discover connections from the set of resources
200+
wls, conns, errs := ps.extractConnections(resAcc)
201201
fileErrors = append(fileErrors, errs...)
202202
return wls, conns, fileErrors
203203
}
204204

205-
func (ps *PoliciesSynthesizer) extractConnections(resFinder *resourceFinder) (
205+
func (ps *PoliciesSynthesizer) extractConnections(resAcc *resourceAccumulator) (
206206
[]*Resource, []*Connections, []FileProcessingError) {
207-
if len(resFinder.workloads) == 0 {
207+
if len(resAcc.workloads) == 0 {
208208
return nil, nil, appendAndLogNewError(nil, noK8sResourcesFound(), ps.logger)
209209
}
210210

211211
// Inline configmaps values as workload envs
212-
fileErrors := resFinder.inlineConfigMapRefsAsEnvs()
212+
fileErrors := resAcc.inlineConfigMapRefsAsEnvs()
213213
if stopProcessing(ps.stopOnError, fileErrors) {
214214
return nil, nil, fileErrors
215215
}
216216

217-
resFinder.exposeServices()
217+
resAcc.exposeServices()
218218

219219
// Discover all connections between resources
220-
connections := discoverConnections(resFinder.workloads, resFinder.services, ps.logger)
221-
return resFinder.workloads, connections, fileErrors
220+
connections := discoverConnections(resAcc.workloads, resAcc.services, ps.logger)
221+
return resAcc.workloads, connections, fileErrors
222222
}
223223

224224
func hasFatalError(errs []FileProcessingError) error {

0 commit comments

Comments
 (0)