Skip to content

Commit 7a27192

Browse files
mrutkowsmrutkows
authored
Support offline validation (no network) using only built-in, local schema (#114)
* Enable loading of remote JSON schemas on --force flag Signed-off-by: Matt Rutkowski <[email protected]> * Enable loading of remote JSON schemas on --force flag Signed-off-by: Matt Rutkowski <[email protected]> * Change use of accent char. to single quote char. in log msgs. Signed-off-by: Matt Rutkowski <[email protected]> * Support offline validation (no network) using only built-in, local schema Signed-off-by: Matt Rutkowski <[email protected]> * Load/compile dependency schemas for any supported schema Signed-off-by: Matt Rutkowski <[email protected]> * Assure v1.2 and v1.3 schemas include their dep. schemas Signed-off-by: Matt Rutkowski <[email protected]> * Fix README duplication Signed-off-by: Matt Rutkowski <[email protected]> * Fix README duplication Signed-off-by: Matt Rutkowski <[email protected]> * Add off-line validation desc. for supported schemas to README Signed-off-by: Matt Rutkowski <[email protected]> * Add off-line validation desc. for supported schemas to README Signed-off-by: Matt Rutkowski <[email protected]> * Rename local function to better describe what it does Signed-off-by: Matt Rutkowski <[email protected]> * Provide more INFO messages around dep. schema loading and compile Signed-off-by: Matt Rutkowski <[email protected]> * Remove temporary files generated during manual testing Signed-off-by: Matt Rutkowski <[email protected]> --------- Signed-off-by: Matt Rutkowski <[email protected]>
1 parent 39bfb28 commit 7a27192

File tree

9 files changed

+1240
-40
lines changed

9 files changed

+1240
-40
lines changed

‎.vscode/launch.json

+21
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,27 @@
55
"version": "0.2.0",
66
"configurations": [
77

8+
{
9+
"showGlobalVariables": true,
10+
"name": "Debug: validate",
11+
"type": "go",
12+
"request": "launch",
13+
"mode": "debug",
14+
"program": "main.go", // "program": "${file}",
15+
"args": ["validate", "-i", "examples/cyclonedx/SBOM/protonmail-webclient-v4-0912dff/bom.json"],
16+
"dlvFlags": ["--check-go-version=false"]
17+
},
18+
{
19+
"showGlobalVariables": true,
20+
"name": "Debug: validate (offline)",
21+
"type": "go",
22+
"request": "launch",
23+
"mode": "debug",
24+
"program": "main.go", // "program": "${file}",
25+
"args": ["validate", "-i", "test/cyclonedx/cdx-1-5-mature-example-1.json"],
26+
"dlvFlags": ["--check-go-version=false"]
27+
},
28+
829
{
930
"showGlobalVariables": true,
1031
"name": "Debug: validate",

‎README.md

+8-2
Original file line numberDiff line numberDiff line change
@@ -318,11 +318,17 @@ See each command's section for contextual examples of the `--where` flag filter
318318

319319
### Validate
320320

321-
This command will parse standardized SBOMs and validate it against its declared format and version (e.g., SPDX 2.3, CycloneDX 1.6). Custom variants of standard JSON schemas can be used for validation by supplying the `--variant` name as a flag. Explicit JSON schemas can be specified using the `--force` flag.
321+
This command will parse standardized SBOMs and validate it against its declared format and version (e.g., SPDX 2.3, CycloneDX 1.6).
322+
323+
- Custom variants of standard JSON schemas can be used for validation by supplying the `--variant` name as a flag.
324+
- Explicit JSON schemas can be specified using the `--force` flag.
322325

323326
#### Validating using supported schemas
324327

325-
Use the [schema](#schema) command to list supported schemas formats, versions and variants.
328+
Use the [schema](#schema) command to list supported schemas formats, versions and variants.
329+
330+
- A "supported" schema is already **"built-in"** to the utility resources along with any dependent schemas it imports.
331+
- This means that BOM files **can be validated when there is no network connection** to load the schemas from remote locations (a.k.a., *"off-line"* mode).
326332

327333
#### Validating using "custom" schemas
328334

‎cmd/validate.go

+93-17
Original file line numberDiff line numberDiff line change
@@ -181,6 +181,71 @@ func validationError(document *schema.BOM, valid bool, err error) {
181181
getLogger().Info(message)
182182
}
183183

184+
func LoadSchemaDependencies(depSchemaLoader *gojsonschema.SchemaLoader, schemas []schema.FormatSchemaInstance, schemaNames []string) (err error) {
185+
for _, schemaName := range schemaNames {
186+
formatSchemaInstance, errMatch := schema.FindMatchingFormatSchemaInstance(
187+
schemas, schemaName)
188+
if errMatch != nil {
189+
return fmt.Errorf("schema '%s' match not found in resources: '%s'", schemaName, schema.SCHEMA_FORMAT_COMMON)
190+
}
191+
getLogger().Debugf("Found: '%s': %v", schemaName, formatSchemaInstance)
192+
193+
getLogger().Infof("Added schema '%s' to loader:...", formatSchemaInstance.File)
194+
err = AddDependencySchemaToLoader(depSchemaLoader, formatSchemaInstance)
195+
if err != nil {
196+
return
197+
}
198+
}
199+
return
200+
}
201+
202+
func AddDependencySchemaToLoader(depSchemaLoader *gojsonschema.SchemaLoader, formatSchemaInstance schema.FormatSchemaInstance) (err error) {
203+
getLogger().Debugf("Reading schema: '%s'...", formatSchemaInstance.File)
204+
bSchema, errRead := resources.BOMSchemaFiles.ReadFile(formatSchemaInstance.File)
205+
206+
if errRead != nil {
207+
return errRead
208+
}
209+
getLogger().Tracef("schema: %s", bSchema)
210+
sharedSchemaLoader := gojsonschema.NewBytesLoader(bSchema)
211+
depSchemaLoader.AddSchema(formatSchemaInstance.Url, sharedSchemaLoader)
212+
return
213+
}
214+
215+
func LoadCompileSchemaDependencies(
216+
bomSchemaLoader gojsonschema.JSONLoader,
217+
bomSchemaInstance schema.FormatSchemaInstance,
218+
bomSchemaDependencies []string,
219+
) (jsonBOMSchema *gojsonschema.Schema, err error) {
220+
221+
if len(bomSchemaDependencies) > 0 {
222+
getLogger().Infof("Found schema dependencies: %v", bomSchemaDependencies)
223+
// Create a schema loader we will add all dep. schemas into
224+
depSchemaLoader := gojsonschema.NewSchemaLoader()
225+
226+
// Load common schema from application resources
227+
var commonSchemas schema.FormatSchema
228+
commonSchemas, err = SupportedFormatConfig.FindMatchingFormatSchema(schema.SCHEMA_FORMAT_COMMON)
229+
if err != nil {
230+
return
231+
}
232+
getLogger().Tracef("Found '%s' schemas: %v", schema.SCHEMA_FORMAT_COMMON, commonSchemas)
233+
234+
err = LoadSchemaDependencies(depSchemaLoader, commonSchemas.Schemas, bomSchemaDependencies)
235+
if err != nil {
236+
return
237+
}
238+
239+
// Compile BOM schema (JSON) with the dependency schemas and return it
240+
getLogger().Infof("Compiling schema: '%s'...", bomSchemaInstance.File)
241+
jsonBOMSchema, err = depSchemaLoader.Compile(bomSchemaLoader)
242+
if err != nil {
243+
return
244+
}
245+
}
246+
return
247+
}
248+
184249
func Validate(writer io.Writer, persistentFlags utils.PersistentCommandFlags, validateFlags utils.ValidateCommandFlags) (valid bool, bom *schema.BOM, schemaErrors []gojsonschema.ResultError, err error) {
185250
getLogger().Enter()
186251
defer getLogger().Exit()
@@ -213,8 +278,8 @@ func Validate(writer io.Writer, persistentFlags utils.PersistentCommandFlags, va
213278

214279
// Create a loader for the BOM (JSON) document
215280
var documentLoader gojsonschema.JSONLoader
216-
var schemaLoader gojsonschema.JSONLoader
217-
var errRead error
281+
var jsonBOMSchemaLoader gojsonschema.JSONLoader
282+
var errRead, errLoadCompile error
218283
var bSchema, bDocument []byte
219284

220285
if bDocument = bom.GetRawBytes(); len(bDocument) > 0 {
@@ -233,14 +298,15 @@ func Validate(writer io.Writer, persistentFlags utils.PersistentCommandFlags, va
233298
return INVALID, bom, schemaErrors, fmt.Errorf("unable to load document: '%s'", bom.GetFilename())
234299
}
235300

301+
// Regardless of how or where we load JSON schemas from the final
302+
// we define the overall Schema object used to validate the BOM document
303+
var jsonBOMSchema *gojsonschema.Schema
236304
schemaName := bom.SchemaInfo.File
237305

238306
// If caller "forced" a specific schema file (version), load it instead of
239307
// any SchemaInfo found in config.json
240-
// TODO: support remote schema load (via URL) with a flag (default should always be local file for security)
241308
forcedSchemaFile := validateFlags.ForcedJsonSchemaFile
242309
if forcedSchemaFile != "" {
243-
244310
if !isValidURIPrefix(forcedSchemaFile) {
245311
// attempt to load as a local file
246312
forcedSchemaFile = "file://" + forcedSchemaFile
@@ -254,7 +320,7 @@ func Validate(writer io.Writer, persistentFlags utils.PersistentCommandFlags, va
254320
}
255321

256322
getLogger().Infof("Loading schema from '--force' flag: '%s'...", forcedSchemaFile)
257-
schemaLoader = gojsonschema.NewReferenceLoader(forcedSchemaFile)
323+
jsonBOMSchemaLoader = gojsonschema.NewReferenceLoader(forcedSchemaFile)
258324
getLogger().Infof("Validating document using forced schema (i.e., '--force %s')", forcedSchemaFile)
259325
} else {
260326
// Load the matching JSON schema (format, version and variant) from embedded resources
@@ -268,10 +334,19 @@ func Validate(writer io.Writer, persistentFlags utils.PersistentCommandFlags, va
268334
return INVALID, bom, schemaErrors, errRead
269335
}
270336

271-
schemaLoader = gojsonschema.NewBytesLoader(bSchema)
337+
// Create a schema loader for the primary BOM schema file
338+
jsonBOMSchemaLoader = gojsonschema.NewBytesLoader(bSchema)
339+
340+
// If the BOM schema has $refs to other schemas, attempt to load and compile
341+
// them from those included as built-in resources
342+
jsonBOMSchema, errLoadCompile = LoadCompileSchemaDependencies(jsonBOMSchemaLoader, bom.SchemaInfo, bom.SchemaInfo.Dependencies)
343+
if err != nil {
344+
return INVALID, bom, schemaErrors, errLoadCompile
345+
}
272346
}
273347

274-
if schemaLoader == nil {
348+
// At this point we should have a BOM schema loader
349+
if jsonBOMSchemaLoader == nil {
275350
// we force result to INVALID as any errors from the library means
276351
// we could NOT actually confirm the input documents validity
277352
return INVALID, bom, schemaErrors, fmt.Errorf("unable to read schema: '%s'", schemaName)
@@ -280,27 +355,28 @@ func Validate(writer io.Writer, persistentFlags utils.PersistentCommandFlags, va
280355
// create a reusable schema object (TODO: validate multiple documents)
281356
var errLoad error = nil
282357
const RETRY int = 3
283-
var jsonBOMSchema *gojsonschema.Schema
284358

285359
// we force result to INVALID as any errors from the library means
286360
// we could NOT actually confirm the input documents validity
287361
// WARNING: if schemas reference "remote" schemas which are loaded
288362
// over http... then there is a chance of 503 errors (as the pkg. loads
289363
// externally referenced schemas over network)... attempt fixed retry...
290-
for i := 0; i < RETRY; i++ {
291-
jsonBOMSchema, errLoad = gojsonschema.NewSchema(schemaLoader)
364+
if jsonBOMSchema == nil {
365+
for i := 0; i < RETRY; i++ {
366+
jsonBOMSchema, errLoad = gojsonschema.NewSchema(jsonBOMSchemaLoader)
292367

293-
if errLoad == nil {
294-
break
368+
if errLoad == nil {
369+
break
370+
}
371+
getLogger().Warningf("unable to load referenced schema over HTTP: \"%v\"\n retrying...", errLoad)
295372
}
296-
getLogger().Warningf("unable to load referenced schema over HTTP: \"%v\"\n retrying...", errLoad)
297-
}
298373

299-
if errLoad != nil {
300-
return INVALID, bom, schemaErrors, fmt.Errorf("unable to load schema: '%s'", schemaName)
374+
if errLoad != nil {
375+
return INVALID, bom, schemaErrors, fmt.Errorf("unable to load schema: `%s`", schemaName)
376+
}
301377
}
302378

303-
getLogger().Infof("Schema '%s' loaded.", schemaName)
379+
getLogger().Infof("Schema '%s' loaded", schemaName)
304380

305381
// Validate against the schema and save result determination
306382
getLogger().Infof("Validating '%s'...", bom.GetFilenameInterpolated())

‎resources/config/config.json

+44-9
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,31 @@
11
{
22
"formats": [
3+
{
4+
"canonicalName": "common",
5+
"propertyKeyFormat": "",
6+
"propertyKeyVersion": "version",
7+
"propertyValueFormat": "latest",
8+
"schemas": [
9+
{
10+
"version": "",
11+
"variant": "",
12+
"name": "jsf-0.82.schema.json",
13+
"file": "schema/cyclonedx/common/jsf-0.82.schema.json",
14+
"development": "",
15+
"url": "http://cyclonedx.org/schema/jsf-0.82.schema.json",
16+
"default": false
17+
},
18+
{
19+
"version": "",
20+
"variant": "",
21+
"name": "spdx.schema.json",
22+
"file": "schema/cyclonedx/common/spdx.schema.json",
23+
"development": "",
24+
"url": "http://cyclonedx.org/schema/spdx.schema.json",
25+
"default": false
26+
}
27+
]
28+
},
329
{
430
"canonicalName": "SPDX",
531
"propertyKeyFormat": "SPDXID",
@@ -57,7 +83,8 @@
5783
"file": "schema/cyclonedx/1.2/bom-1.2.schema.json",
5884
"development": "https://github.com/CycloneDX/specification/blob/master/schema/bom-1.2.schema.json",
5985
"url": "https://raw.githubusercontent.com/CycloneDX/specification/master/schema/bom-1.2.schema.json",
60-
"default": false
86+
"default": false,
87+
"dependencies": ["spdx.schema.json"]
6188
},
6289
{
6390
"version": "1.2",
@@ -66,7 +93,8 @@
6693
"file": "schema/cyclonedx/1.2/bom-1.2-strict.schema.json",
6794
"development": "https://github.com/CycloneDX/specification/blob/master/schema/bom-1.2-strict.schema.json",
6895
"url": "https://raw.githubusercontent.com/CycloneDX/specification/master/schema/bom-1.2-strict.schema.json",
69-
"default": false
96+
"default": false,
97+
"dependencies": ["spdx.schema.json"]
7098
},
7199
{
72100
"version": "1.3",
@@ -75,7 +103,8 @@
75103
"file": "schema/cyclonedx/1.3/bom-1.3.schema.json",
76104
"development": "https://github.com/CycloneDX/specification/blob/master/schema/bom-1.3.schema.json",
77105
"url": "https://raw.githubusercontent.com/CycloneDX/specification/master/schema/bom-1.3.schema.json",
78-
"default": false
106+
"default": false,
107+
"dependencies": ["spdx.schema.json"]
79108
},
80109
{
81110
"version": "1.3",
@@ -84,7 +113,8 @@
84113
"file": "schema/cyclonedx/1.3/bom-1.3-strict.schema.json",
85114
"development": "https://github.com/CycloneDX/specification/blob/master/schema/bom-1.3-strict.schema.json",
86115
"url": "https://raw.githubusercontent.com/CycloneDX/specification/master/schema/bom-1.3-strict.schema.json",
87-
"default": false
116+
"default": false,
117+
"dependencies": ["spdx.schema.json"]
88118
},
89119
{
90120
"version": "1.4",
@@ -93,7 +123,8 @@
93123
"file": "schema/cyclonedx/1.4/bom-1.4.schema.json",
94124
"development": "https://github.com/CycloneDX/specification/blob/master/schema/bom-1.4.schema.json",
95125
"url": "https://raw.githubusercontent.com/CycloneDX/specification/master/schema/bom-1.4.schema.json",
96-
"default": false
126+
"default": false,
127+
"dependencies": ["jsf-0.82.schema.json", "spdx.schema.json"]
97128
},
98129
{
99130
"version": "1.5",
@@ -102,7 +133,8 @@
102133
"file": "schema/cyclonedx/1.5/bom-1.5.schema.json",
103134
"development": "https://github.com/CycloneDX/specification/blob/master/schema/bom-1.5.schema.json",
104135
"url": "https://raw.githubusercontent.com/CycloneDX/specification/master/schema/bom-1.5.schema.json",
105-
"default": false
136+
"default": false,
137+
"dependencies": ["jsf-0.82.schema.json", "spdx.schema.json"]
106138
},
107139
{
108140
"version": "1.6",
@@ -111,7 +143,8 @@
111143
"file": "schema/cyclonedx/1.6/bom-1.6.schema.json",
112144
"development": "https://github.com/CycloneDX/specification/blob/master/schema/bom-1.6.schema.json",
113145
"url": "https://raw.githubusercontent.com/CycloneDX/specification/master/schema/bom-1.6.schema.json",
114-
"default": true
146+
"default": true,
147+
"dependencies": ["jsf-0.82.schema.json", "spdx.schema.json"]
115148
},
116149
{
117150
"version": "1.3",
@@ -120,7 +153,8 @@
120153
"file": "schema/test/bom-1.3-custom.schema.json",
121154
"development":"https://github.com/CycloneDX/sbom-utility/blob/main/resources/schema/test/bom-1.3-custom.schema.json",
122155
"url": "",
123-
"default": false
156+
"default": false,
157+
"dependencies": ["spdx.schema.json"]
124158
},
125159
{
126160
"version": "1.4",
@@ -129,7 +163,8 @@
129163
"file": "schema/test/bom-1.4-custom.schema.json",
130164
"development":"https://github.com/CycloneDX/sbom-utility/blob/main/resources/schema/test/bom-1.4-custom.schema.json",
131165
"url": "",
132-
"default": false
166+
"default": false,
167+
"dependencies": ["jsf-0.82.schema.json", "spdx.schema.json"]
133168
}
134169
]
135170
}

‎resources/resources.go

+4-4
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ func LoadConfigFile(baseFilename string) (bData []byte, err error) {
3737
return
3838
}
3939

40-
func LoadSchemaFile(baseFilename string) (bData []byte, err error) {
41-
bData, err = ConfigFiles.ReadFile(RESOURCES_SCHEMA_DIR + baseFilename)
42-
return
43-
}
40+
// func LoadSchemaFile(baseFilename string) (bData []byte, err error) {
41+
// bData, err = BOMSchemaFiles.ReadFile(RESOURCES_SCHEMA_DIR + baseFilename)
42+
// return
43+
// }

0 commit comments

Comments
 (0)