Skip to content

Commit e421ef1

Browse files
mrutkowsmrutkows
authored
Improve marshal performance by using pointers within CycloneDX Vulnerability data structures (#65)
* Improve performance by using pointers within CycloneDX Vulnerability structs Signed-off-by: Matt Rutkowski <[email protected]> * Migrate Vuln. struct members to pointers and update marhsal routines Signed-off-by: Matt Rutkowski <[email protected]> * Introduce const for JSON indent. spacing and set to conventional defaults Signed-off-by: Matt Rutkowski <[email protected]> --------- Signed-off-by: Matt Rutkowski <[email protected]>
1 parent a30779e commit e421ef1

13 files changed

+404
-267
lines changed

‎cmd/query_test.go

+1-10
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ func innerQuery(t *testing.T, filename string, queryRequest *common.QueryRequest
6161
}
6262

6363
// This will print results ONLY if --quiet mode is `false`
64-
printResult(result)
64+
printMarshaledResultOnlyIfNotQuiet(result)
6565
return
6666
}
6767

@@ -122,15 +122,6 @@ func VerifySelectedFieldsInJsonMap(t *testing.T, keys []string, results interfac
122122
return
123123
}
124124

125-
func printResult(iResult interface{}) {
126-
if !*TestLogQuiet {
127-
// Format results in JSON
128-
fResult, _ := utils.MarshalAnyToFormattedJsonString(iResult)
129-
// Output the JSON data directly to stdout (not subject to log-level)
130-
fmt.Printf("%s\n", fResult)
131-
}
132-
}
133-
134125
// ----------------------------------------
135126
// Command flag tests
136127
// ----------------------------------------

‎cmd/root_test.go

+10
Original file line numberDiff line numberDiff line change
@@ -283,3 +283,13 @@ func bufferContainsValues(buffer bytes.Buffer, values ...string) bool {
283283
}
284284
return true
285285
}
286+
287+
// TODO: find a better way using some log package feature
288+
func printMarshaledResultOnlyIfNotQuiet(iResult interface{}) {
289+
if !*TestLogQuiet {
290+
// Format results in JSON
291+
fResult, _ := utils.MarshalAnyToFormattedJsonString(iResult)
292+
// Output the JSON data directly to stdout (not subject to log-level)
293+
fmt.Printf("%s\n", fResult)
294+
}
295+
}

‎cmd/trim.go

+12-4
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,14 @@ const (
3434
FLAG_TRIM_MAP_KEYS = "keys"
3535
)
3636

37+
// TODO: make flag configurable:
38+
// NOTE: 4-space indent is accepted convention:
39+
// https://docs.openstack.org/doc-contrib-guide/json-conv.html
40+
const (
41+
TRIM_OUTPUT_PREFIX = ""
42+
TRIM_OUTPUT_INDENT = " "
43+
)
44+
3745
// flag help (translate)
3846
const (
3947
FLAG_TRIM_OUTPUT_FORMAT_HELP = "format output using the specified type"
@@ -207,12 +215,12 @@ func Trim(writer io.Writer, persistentFlags utils.PersistentCommandFlags, trimFl
207215
getLogger().Infof("Outputting listing (`%s` format)...", format)
208216
switch format {
209217
case FORMAT_JSON:
210-
err = document.EncodeAsFormattedJSON(writer, "", " ")
218+
err = document.EncodeAsFormattedJSON(writer, TRIM_OUTPUT_PREFIX, TRIM_OUTPUT_INDENT)
211219
default:
212220
// Default to Text output for anything else (set as flag default)
213-
getLogger().Warningf("Stats not supported for `%s` format; defaulting to `%s` format...",
214-
format, FORMAT_TEXT)
215-
err = document.EncodeAsFormattedJSON(writer, "", " ")
221+
getLogger().Warningf("Trim not supported for `%s` format; defaulting to `%s` format...",
222+
format, FORMAT_JSON)
223+
err = document.EncodeAsFormattedJSON(writer, TRIM_OUTPUT_PREFIX, TRIM_OUTPUT_INDENT)
216224
}
217225

218226
return

‎cmd/trim_test.go

+22
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ const (
3737
TEST_TRIM_CDX_1_4_ENCODED_CHARS = "test/trim/trim-cdx-1-4-sample-encoded-chars.sbom.json"
3838
TEST_TRIM_CDX_1_4_SAMPLE_XXL_1 = "test/trim/trim-cdx-1-4-sample-xxl-1.sbom.json"
3939
TEST_TRIM_CDX_1_5_SAMPLE_SMALL_COMPS_ONLY = "test/trim/trim-cdx-1-5-sample-small-components-only.sbom.json"
40+
TEST_TRIM_CDX_1_4_SAMPLE_VEX = "test/trim/trim-cdx-1-4-sample-vex.json"
4041
TEST_TRIM_CDX_1_5_SAMPLE_MEDIUM_1 = "test/trim/trim-cdx-1-5-sample-medium-1.sbom.json"
4142
)
4243

@@ -352,3 +353,24 @@ func TestTrimCdx15FooFromTools(t *testing.T) {
352353
t.Error(fmt.Errorf("invalid trim result: string not found: %s", TEST_STRING_1))
353354
}
354355
}
356+
357+
func TestTrimCdx14SourceFromVulnerabilities(t *testing.T) {
358+
ti := NewTrimTestInfoBasic(TEST_TRIM_CDX_1_4_SAMPLE_VEX, nil)
359+
ti.Keys = append(ti.Keys, "source")
360+
ti.FromPaths = []string{"vulnerabilities"}
361+
ti.TestOutputVariantName = utils.GetCallerFunctionName(2)
362+
ti.OutputFile = ti.CreateTemporaryFilename(TEST_TRIM_CDX_1_4_SAMPLE_VEX)
363+
364+
buffer, _, err := innerTestTrim(t, ti)
365+
s := buffer.String()
366+
if err != nil {
367+
getLogger().Debugf("result: %s", s)
368+
t.Error(err)
369+
}
370+
371+
// Assure JSON map does not contain the trimmed key(s)
372+
err = VerifyTrimOutputFileResult(t, ti, ti.Keys, ti.FromPaths[0])
373+
if err != nil {
374+
t.Error(err)
375+
}
376+
}

‎cmd/vulnerability.go

+1-144
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@ import (
2222
"encoding/csv"
2323
"fmt"
2424
"io"
25-
"reflect"
2625
"sort"
2726
"strings"
2827
"text/tabwriter"
@@ -260,156 +259,14 @@ func loadDocumentVulnerabilities(document *schema.BOM, whereFilters []common.Whe
260259
// Hash all components found in the (root).components[] (+ "nested" components)
261260
pVulnerabilities := document.GetCdxVulnerabilities()
262261
if pVulnerabilities != nil && len(*pVulnerabilities) > 0 {
263-
if err = hashVulnerabilities(document, *pVulnerabilities, whereFilters); err != nil {
262+
if err = document.HashVulnerabilities(*pVulnerabilities, whereFilters); err != nil {
264263
return
265264
}
266265
}
267266

268267
return
269268
}
270269

271-
// We need to hash our own informational structure around the CDX data in order
272-
// to simplify --where queries to command line users
273-
func hashVulnerabilities(bom *schema.BOM, vulnerabilities []schema.CDXVulnerability, whereFilters []common.WhereFilter) (err error) {
274-
getLogger().Enter()
275-
defer getLogger().Exit(err)
276-
277-
for _, cdxVulnerability := range vulnerabilities {
278-
_, err = hashVulnerability(bom, cdxVulnerability, whereFilters)
279-
if err != nil {
280-
return
281-
}
282-
}
283-
return
284-
}
285-
286-
// Hash a CDX Component and recursively those of any "nested" components
287-
// TODO we should WARN if version is not a valid semver (e.g., examples/cyclonedx/BOM/laravel-7.12.0/bom.1.3.json)
288-
func hashVulnerability(bom *schema.BOM, cdxVulnerability schema.CDXVulnerability, whereFilters []common.WhereFilter) (vi *schema.VulnerabilityInfo, err error) {
289-
getLogger().Enter()
290-
defer getLogger().Exit(err)
291-
var vulnInfo schema.VulnerabilityInfo
292-
vi = &vulnInfo
293-
294-
if reflect.DeepEqual(cdxVulnerability, schema.CDXVulnerability{}) {
295-
err = getLogger().Errorf("invalid vulnerability info: missing or empty : %v ", cdxVulnerability)
296-
return
297-
}
298-
299-
if cdxVulnerability.Id == "" {
300-
getLogger().Warningf("vulnerability missing required value `id` : %v ", cdxVulnerability)
301-
}
302-
303-
if cdxVulnerability.Published == "" {
304-
getLogger().Warningf("vulnerability (`%s`) missing `published` date", cdxVulnerability.Id)
305-
}
306-
307-
if cdxVulnerability.Created == "" {
308-
getLogger().Warningf("vulnerability (`%s`) missing `created` date", cdxVulnerability.Id)
309-
}
310-
311-
if len(cdxVulnerability.Ratings) == 0 {
312-
getLogger().Warningf("vulnerability (`%s`) missing `ratings`", cdxVulnerability.Id)
313-
}
314-
315-
// hash any component w/o a license using special key name
316-
vulnInfo.Vulnerability = cdxVulnerability
317-
if cdxVulnerability.BOMRef != nil {
318-
vulnInfo.BOMRef = cdxVulnerability.BOMRef.String()
319-
}
320-
vulnInfo.Id = cdxVulnerability.Id
321-
322-
// Truncate dates from 2023-02-02T00:00:00.000Z to 2023-02-02
323-
// Note: if validation errors are found by the "truncate" function,
324-
// it will emit an error and return the original (failing) value
325-
dateTime, _ := utils.TruncateTimeStampISO8601Date(cdxVulnerability.Created)
326-
vulnInfo.Created = dateTime
327-
328-
dateTime, _ = utils.TruncateTimeStampISO8601Date(cdxVulnerability.Published)
329-
vulnInfo.Published = dateTime
330-
331-
dateTime, _ = utils.TruncateTimeStampISO8601Date(cdxVulnerability.Updated)
332-
vulnInfo.Updated = dateTime
333-
334-
dateTime, _ = utils.TruncateTimeStampISO8601Date(cdxVulnerability.Rejected)
335-
vulnInfo.Rejected = dateTime
336-
337-
vulnInfo.Description = cdxVulnerability.Description
338-
339-
// Source object: retrieve report fields from nested objects
340-
if cdxVulnerability.Source != nil {
341-
source := *cdxVulnerability.Source
342-
vulnInfo.Source = source
343-
vulnInfo.SourceName = source.Name
344-
vulnInfo.SourceUrl = source.Url
345-
}
346-
347-
// TODO: replace empty Analysis values with "UNDEFINED"
348-
vulnInfo.AnalysisState = cdxVulnerability.Analysis.State
349-
if vulnInfo.AnalysisState == "" {
350-
vulnInfo.AnalysisState = schema.VULN_ANALYSIS_STATE_EMPTY
351-
}
352-
353-
vulnInfo.AnalysisJustification = cdxVulnerability.Analysis.Justification
354-
if vulnInfo.AnalysisJustification == "" {
355-
vulnInfo.AnalysisJustification = schema.VULN_ANALYSIS_STATE_EMPTY
356-
}
357-
vulnInfo.AnalysisResponse = cdxVulnerability.Analysis.Response
358-
if len(vulnInfo.AnalysisResponse) == 0 {
359-
vulnInfo.AnalysisResponse = []string{schema.VULN_ANALYSIS_STATE_EMPTY}
360-
}
361-
362-
// Convert []int to []string for --where filter
363-
// TODO see if we can eliminate this conversion and handle while preparing report data
364-
// as this SHOULD appear there as []interface{}
365-
if len(cdxVulnerability.Cwes) > 0 {
366-
vulnInfo.CweIds = strings.Fields(strings.Trim(fmt.Sprint(cdxVulnerability.Cwes), "[]"))
367-
}
368-
369-
// CVSS Score Qualitative Rating
370-
// 0.0 None
371-
// 0.1 – 3.9 Low
372-
// 4.0 – 6.9 Medium
373-
// 7.0 – 8.9 High
374-
// 9.0 – 10.0 Critical
375-
376-
// TODO: if summary report, see if more than one severity can be shown without clogging up column data
377-
numRatings := len(cdxVulnerability.Ratings)
378-
if numRatings > 0 {
379-
//var sourceMatch int
380-
for _, rating := range cdxVulnerability.Ratings {
381-
// defer to same source as the top-level vuln. declares
382-
fSeverity := fmt.Sprintf("%s: %v (%s)", rating.Method, rating.Score, rating.Severity)
383-
// give listing priority to ratings that matches top-level vuln. reporting source
384-
if rating.Source.Name == cdxVulnerability.Source.Name {
385-
// prepend to slice
386-
vulnInfo.CvssSeverity = append([]string{fSeverity}, vulnInfo.CvssSeverity...)
387-
continue
388-
}
389-
vulnInfo.CvssSeverity = append(vulnInfo.CvssSeverity, fSeverity)
390-
}
391-
392-
} else {
393-
// Set first entry to empty value (i.e., "none")
394-
vulnInfo.CvssSeverity = append(vulnInfo.CvssSeverity, schema.VULN_RATING_EMPTY)
395-
}
396-
397-
var match bool = true
398-
if len(whereFilters) > 0 {
399-
mapVulnInfo, _ := utils.MarshalStructToJsonMap(vulnInfo)
400-
match, _ = whereFilterMatch(mapVulnInfo, whereFilters)
401-
}
402-
403-
if match {
404-
bom.VulnerabilityMap.Put(vulnInfo.Id, vulnInfo)
405-
406-
getLogger().Tracef("Put: %s (`%s`), `%s`)",
407-
vulnInfo.Id, vulnInfo.Description, vulnInfo.BOMRef)
408-
}
409-
410-
return
411-
}
412-
413270
// NOTE: This list is NOT de-duplicated
414271
// TODO: Add a --no-title flag to skip title output
415272
func DisplayVulnListText(bom *schema.BOM, output io.Writer, flags utils.VulnerabilityCommandFlags) {

‎cmd/vulnerability_test.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -232,7 +232,6 @@ func TestVulnListCdx13JSON(t *testing.T) {
232232
testInfo.ResultExpectedLineCount = 185
233233
result, _, _ := innerTestVulnList(t, testInfo, VULN_TEST_DEFAULT_FLAGS)
234234
getLogger().Debugf("result:\n%s", result.String())
235-
//fmt.Printf("result:\n%s", result.String())
236235
}
237236

238237
// -------------------------------------------
@@ -254,7 +253,8 @@ func TestVulnListTextCdx14WhereClauseAndResultsByIdStartsWith(t *testing.T) {
254253
nil)
255254
testInfo.ResultLineContainsValues = TEST_OUTPUT_CONTAINS
256255
testInfo.ResultLineContainsValuesAtLineNum = 2
257-
innerTestVulnList(t, testInfo, VULN_TEST_DEFAULT_FLAGS)
256+
result, _, _ := innerTestVulnList(t, testInfo, VULN_TEST_DEFAULT_FLAGS)
257+
getLogger().Debugf("result:\n%s", result.String())
258258
}
259259

260260
func TestVulnListTextCdx14WhereClauseDescContains(t *testing.T) {

‎schema/bom_hash.go

+21-15
Original file line numberDiff line numberDiff line change
@@ -346,7 +346,7 @@ func (bom *BOM) HashVulnerability(cdxVulnerability CDXVulnerability, whereFilter
346346
getLogger().Warningf("vulnerability (`%s`) missing `created` date", cdxVulnerability.Id)
347347
}
348348

349-
if len(cdxVulnerability.Ratings) == 0 {
349+
if cdxVulnerability.Ratings == nil || len(*cdxVulnerability.Ratings) == 0 {
350350
getLogger().Warningf("vulnerability (`%s`) missing `ratings`", cdxVulnerability.Id)
351351
}
352352

@@ -383,24 +383,32 @@ func (bom *BOM) HashVulnerability(cdxVulnerability CDXVulnerability, whereFilter
383383
}
384384

385385
// TODO: replace empty Analysis values with "UNDEFINED"
386-
vulnInfo.AnalysisState = cdxVulnerability.Analysis.State
387-
if vulnInfo.AnalysisState == "" {
388-
vulnInfo.AnalysisState = VULN_ANALYSIS_STATE_EMPTY
389-
}
386+
if cdxVulnerability.Analysis != nil {
387+
vulnInfo.AnalysisState = cdxVulnerability.Analysis.State
388+
if vulnInfo.AnalysisState == "" {
389+
vulnInfo.AnalysisState = VULN_ANALYSIS_STATE_EMPTY
390+
}
391+
392+
vulnInfo.AnalysisJustification = cdxVulnerability.Analysis.Justification
393+
if vulnInfo.AnalysisJustification == "" {
394+
vulnInfo.AnalysisJustification = VULN_ANALYSIS_STATE_EMPTY
395+
}
390396

391-
vulnInfo.AnalysisJustification = cdxVulnerability.Analysis.Justification
392-
if vulnInfo.AnalysisJustification == "" {
397+
vulnInfo.AnalysisResponse = *cdxVulnerability.Analysis.Response
398+
if len(vulnInfo.AnalysisResponse) == 0 {
399+
vulnInfo.AnalysisResponse = []string{VULN_ANALYSIS_STATE_EMPTY}
400+
}
401+
} else {
402+
vulnInfo.AnalysisState = VULN_ANALYSIS_STATE_EMPTY
393403
vulnInfo.AnalysisJustification = VULN_ANALYSIS_STATE_EMPTY
394-
}
395-
vulnInfo.AnalysisResponse = cdxVulnerability.Analysis.Response
396-
if len(vulnInfo.AnalysisResponse) == 0 {
397404
vulnInfo.AnalysisResponse = []string{VULN_ANALYSIS_STATE_EMPTY}
398405
}
399406

400407
// Convert []int to []string for --where filter
401408
// TODO see if we can eliminate this conversion and handle while preparing report data
402409
// as this SHOULD appear there as []interface{}
403-
if len(cdxVulnerability.Cwes) > 0 {
410+
if cdxVulnerability.Cwes != nil && len(*cdxVulnerability.Cwes) > 0 {
411+
// strip off slice/array brackets
404412
vulnInfo.CweIds = strings.Fields(strings.Trim(fmt.Sprint(cdxVulnerability.Cwes), "[]"))
405413
}
406414

@@ -412,10 +420,9 @@ func (bom *BOM) HashVulnerability(cdxVulnerability CDXVulnerability, whereFilter
412420
// 9.0 – 10.0 Critical
413421

414422
// TODO: if summary report, see if more than one severity can be shown without clogging up column data
415-
numRatings := len(cdxVulnerability.Ratings)
416-
if numRatings > 0 {
423+
if cdxVulnerability.Ratings != nil && len(*cdxVulnerability.Ratings) > 0 {
417424
//var sourceMatch int
418-
for _, rating := range cdxVulnerability.Ratings {
425+
for _, rating := range *cdxVulnerability.Ratings {
419426
// defer to same source as the top-level vuln. declares
420427
fSeverity := fmt.Sprintf("%s: %v (%s)", rating.Method, rating.Score, rating.Severity)
421428
// give listing priority to ratings that matches top-level vuln. reporting source
@@ -426,7 +433,6 @@ func (bom *BOM) HashVulnerability(cdxVulnerability CDXVulnerability, whereFilter
426433
}
427434
vulnInfo.CvssSeverity = append(vulnInfo.CvssSeverity, fSeverity)
428435
}
429-
430436
} else {
431437
// Set first entry to empty value (i.e., "none")
432438
vulnInfo.CvssSeverity = append(vulnInfo.CvssSeverity, VULN_RATING_EMPTY)

‎schema/cyclonedx.go

+6-6
Original file line numberDiff line numberDiff line change
@@ -59,8 +59,8 @@ type CDXBom struct {
5959
// v1.3: added "licenses", "properties"
6060
// v1.5: added "lifecycles"
6161
type CDXMetadata struct {
62-
Timestamp string `json:"timestamp,omitempty"`
63-
Tools interface{} `json:"tools,omitempty"` // v1.2: added.v1.5: "tools" is now an interface{}
62+
Timestamp string `json:"timestamp,omitempty" scvs:"bom:core:timestamp"` // urn:owasp:scvs:bom:core:timestamp
63+
Tools interface{} `json:"tools,omitempty"` // v1.2: added.v1.5: "tools" is now an interface{}
6464
Authors *[]CDXOrganizationalContact `json:"authors,omitempty"`
6565
Component *CDXComponent `json:"component,omitempty"`
6666
Manufacturer *CDXOrganizationalEntity `json:"manufacturer,omitempty"`
@@ -96,10 +96,10 @@ type CDXComponent struct {
9696
Hashes *[]CDXHash `json:"hashes,omitempty"`
9797
Licenses *[]CDXLicenseChoice `json:"licenses,omitempty"`
9898
Copyright string `json:"copyright,omitempty"`
99-
Cpe string `json:"cpe,omitempty"` // See: https://nvd.nist.gov/products/cpe
100-
Purl string `json:"purl,omitempty"` // See: https://github.com/package-url/purl-spec
101-
Swid *CDXSwid `json:"swid,omitempty"` // See: https://www.iso.org/standard/65666.html
102-
Pedigree *CDXPedigree `json:"pedigree,omitempty"` // anon. type
99+
Cpe string `json:"cpe,omitempty"` // See: https://nvd.nist.gov/products/cpe
100+
Purl string `json:"purl,omitempty" scvs:"bom:resource:identifiers:purl"` // See: https://github.com/package-url/purl-spec
101+
Swid *CDXSwid `json:"swid,omitempty"` // See: https://www.iso.org/standard/65666.html
102+
Pedigree *CDXPedigree `json:"pedigree,omitempty"` // anon. type
103103
ExternalReferences *[]CDXExternalReference `json:"externalReferences,omitempty"`
104104
Components *[]CDXComponent `json:"components,omitempty"`
105105
Evidence *CDXComponentEvidence `json:"evidence,omitempty"` // v1.3: added

0 commit comments

Comments
 (0)