feat: collect license evidences

fixes #33

---------

Signed-off-by: Augustus Kling <[email protected]>
Signed-off-by: Jan Kowalleck <[email protected]>
Co-authored-by: Jan Kowalleck <[email protected]>

Author: AugustusKling

README.md

@@ -77,6 +77,8 @@ $ yarn cyclonedx
                            This causes information loss in trade-off shorter PURLs, which might improve ingesting these strings.
   --output-reproducible    Whether to go the extra mile and make the output reproducible.
                            This might result in loss of time- and random-based values.
+  --gather-license-texts   Search for license files in components and include them as license evidence.
+                           This feature is experimental.
   --verbose,-v             Increase the verbosity of messages.
                            Use multiple times to increase the verbosity even more.
 

src/_helpers.ts

@@ -19,6 +19,7 @@ Copyright (c) OWASP Foundation. All Rights Reserved.
 
 import { xfs } from '@yarnpkg/fslib'
 import GitHost from 'hosted-git-info'
+import { extname, parse } from 'path'
 
 export async function writeAllSync (fd: number, data: string): Promise<number> {
   const b = Buffer.from(data)
@@ -59,3 +60,46 @@ export function trySanitizeGitUrl (gitUrl: string): string {
   gitInfo.auth = undefined
   return gitInfo.toString()
 }
+
+// region MIME
+
+export type MimeType = string
+
+const MIME_TEXT_PLAIN: MimeType = 'text/plain'
+
+const MAP_TEXT_EXTENSION_MIME: Readonly<Record<string, MimeType>> = {
+  '': MIME_TEXT_PLAIN,
+  // https://www.iana.org/assignments/media-types/media-types.xhtml
+  '.csv': 'text/csv',
+  '.htm': 'text/html',
+  '.html': 'text/html',
+  '.md': 'text/markdown',
+  '.txt': MIME_TEXT_PLAIN,
+  '.rst': 'text/prs.fallenstein.rst',
+  '.xml': 'text/xml', // not `application/xml` -- our scope is text!
+  // add more mime types above this line. pull-requests welcome!
+  // license-specific files
+  '.license': MIME_TEXT_PLAIN,
+  '.licence': MIME_TEXT_PLAIN
+} as const
+
+export function getMimeForTextFile (filename: string): MimeType | undefined {
+  return MAP_TEXT_EXTENSION_MIME[extname(filename).toLowerCase()]
+}
+
+const LICENSE_FILENAME_BASE = new Set(['licence', 'license'])
+const LICENSE_FILENAME_EXT = new Set([
+  '.apache',
+  '.bsd',
+  '.gpl',
+  '.mit'
+])
+
+export function getMimeForLicenseFile (filename: string): MimeType | undefined {
+  const { name, ext } = parse(filename.toLowerCase())
+  return LICENSE_FILENAME_BASE.has(name) && LICENSE_FILENAME_EXT.has(ext)
+    ? MIME_TEXT_PLAIN
+    : MAP_TEXT_EXTENSION_MIME[ext]
+}
+
+// endregion MIME

src/builders.ts

@@ -19,27 +19,29 @@ Copyright (c) OWASP Foundation. All Rights Reserved.
 
 // import submodules so to prevent load of unused not-tree-shakable dependencies - like 'AJV'
 import type { FromNodePackageJson as PJB } from '@cyclonedx/cyclonedx-library/Builders'
-import { ComponentType, ExternalReferenceType, LicenseAcknowledgement } from '@cyclonedx/cyclonedx-library/Enums'
+import { AttachmentEncoding, ComponentType, ExternalReferenceType, LicenseAcknowledgement } from '@cyclonedx/cyclonedx-library/Enums'
 import type { FromNodePackageJson as PJF } from '@cyclonedx/cyclonedx-library/Factories'
-import { Bom, Component, ExternalReference, type License, Property, Tool } from '@cyclonedx/cyclonedx-library/Models'
+import { Attachment, Bom, Component, ComponentEvidence, ExternalReference, type License, NamedLicense, Property, Tool } from '@cyclonedx/cyclonedx-library/Models'
 import { BomUtility } from '@cyclonedx/cyclonedx-library/Utils'
 import { Cache, type FetchOptions, type Locator, type LocatorHash, type Package, type Project, structUtils, ThrowReport, type Workspace, YarnVersion } from '@yarnpkg/core'
 import { ppath } from '@yarnpkg/fslib'
 import { gitUtils as YarnPluginGitUtils } from '@yarnpkg/plugin-git'
 import normalizePackageData from 'normalize-package-data'
 
 import { getBuildtimeInfo } from './_buildtimeInfo'
-import { isString, tryRemoveSecretsFromUrl, trySanitizeGitUrl } from './_helpers'
+import { getMimeForLicenseFile, isString, tryRemoveSecretsFromUrl, trySanitizeGitUrl } from './_helpers'
 import { wsAnchoredPackage } from './_yarnCompat'
 import { PropertyNames, PropertyValueBool } from './properties'
 
 type ManifestFetcher = (pkg: Package) => Promise<any>
+type LicenseEvidenceFetcher = (pkg: Package) => AsyncGenerator<License>
 
 interface BomBuilderOptions {
   omitDevDependencies?: BomBuilder['omitDevDependencies']
   metaComponentType?: BomBuilder['metaComponentType']
   reproducible?: BomBuilder['reproducible']
   shortPURLs?: BomBuilder['shortPURLs']
+  gatherLicenseTexts?: BomBuilder['gatherLicenseTexts']
 }
 
 export class BomBuilder {
@@ -51,6 +53,7 @@ export class BomBuilder {
   metaComponentType: ComponentType
   reproducible: boolean
   shortPURLs: boolean
+  gatherLicenseTexts: boolean
 
   console: Console
 
@@ -69,13 +72,15 @@ export class BomBuilder {
     this.metaComponentType = options.metaComponentType ?? ComponentType.Application
     this.reproducible = options.reproducible ?? false
     this.shortPURLs = options.shortPURLs ?? false
+    this.gatherLicenseTexts = options.gatherLicenseTexts ?? false
 
     this.console = console_
   }
 
   async buildFromWorkspace (workspace: Workspace): Promise<Bom> {
     // @TODO make switch to disable load from fs
     const fetchManifest: ManifestFetcher = await this.makeManifestFetcher(workspace.project)
+    const fetchLicenseEvidences: LicenseEvidenceFetcher = await this.makeLicenseEvidenceFetcher(workspace.project)
 
     const setLicensesDeclared = function (license: License): void {
       license.acknowledgement = LicenseAcknowledgement.Declared
@@ -118,7 +123,8 @@ export class BomBuilder {
     }
     for await (const component of this.gatherDependencies(
       rootComponent, rootPackage,
-      workspace.project, fetchManifest
+      workspace.project,
+      fetchManifest, fetchLicenseEvidences
     )) {
       component.licenses.forEach(setLicensesDeclared)
 
@@ -162,33 +168,90 @@ export class BomBuilder {
     }
   }
 
+  readonly #LICENSE_FILENAME_PATTERN = /^(?:UN)?LICEN[CS]E|.\.LICEN[CS]E$|^NOTICE$/i
+
+  private async makeLicenseEvidenceFetcher (project: Project): Promise<LicenseEvidenceFetcher> {
+    const fetcher = project.configuration.makeFetcher()
+    const fetcherOptions: FetchOptions = {
+      project,
+      fetcher,
+      cache: await Cache.find(project.configuration),
+      checksums: project.storedChecksums,
+      report: new ThrowReport(),
+      cacheOptions: { skipIntegrityCheck: true }
+    }
+    const LICENSE_FILENAME_PATTERN = this.#LICENSE_FILENAME_PATTERN
+    return async function * (pkg: Package): AsyncGenerator<License> {
+      const { packageFs, prefixPath, releaseFs } = await fetcher.fetch(pkg, fetcherOptions)
+      try {
+        // option `withFileTypes:true` is not supported and causes crashes
+        const files = packageFs.readdirSync(prefixPath)
+        for (const file of files) {
+          if (!LICENSE_FILENAME_PATTERN.test(file)) {
+            continue
+          }
+
+          const contentType = getMimeForLicenseFile(file)
+          if (contentType === undefined) {
+            continue
+          }
+
+          const fp = ppath.join(prefixPath, file)
+          yield new NamedLicense(
+          `file: ${file}`,
+          {
+            text: new Attachment(
+              packageFs.readFileSync(fp).toString('base64'),
+              {
+                contentType,
+                encoding: AttachmentEncoding.Base64
+              }
+            )
+          })
+        }
+      } finally {
+        if (releaseFs !== undefined) {
+          releaseFs()
+        }
+      }
+    }
+  }
+
   private async makeComponentFromPackage (
     pkg: Package,
     fetchManifest: ManifestFetcher,
+    fetchLicenseEvidence: LicenseEvidenceFetcher,
     type?: ComponentType | undefined
   ): Promise<Component | false | undefined> {
-    const data = await fetchManifest(pkg)
+    const manifest = await fetchManifest(pkg)
     // the data in the manifest might be incomplete, so lets set the properties that yarn discovered and fixed
     /* eslint-disable-next-line @typescript-eslint/strict-boolean-expressions */
-    data.name = pkg.scope ? `@${pkg.scope}/${pkg.name}` : pkg.name
-    data.version = pkg.version
-    return this.makeComponent(pkg, data, type)
+    manifest.name = pkg.scope ? `@${pkg.scope}/${pkg.name}` : pkg.name
+    manifest.version = pkg.version
+    const component = this.makeComponent(pkg, manifest, type)
+    if (this.gatherLicenseTexts && component instanceof Component) {
+      component.evidence = new ComponentEvidence()
+      for await (const le of fetchLicenseEvidence(pkg)) {
+        component.evidence.licenses.add(le)
+      }
+    }
+    return component
   }
 
-  private makeComponent (locator: Locator, data: any, type?: ComponentType | undefined): Component | false | undefined {
+  private makeComponent (locator: Locator, manifest: any, type?: ComponentType | undefined): Component | false | undefined {
     // work with a deep copy, because `normalizePackageData()` might modify the data
-    const dataC = structuredClonePolyfill(data)
-    normalizePackageData(dataC as normalizePackageData.Input)
+    const manifestC = structuredClonePolyfill(manifest)
+    normalizePackageData(manifestC as normalizePackageData.Input)
     // region fix normalizations
-    if (isString(data.version)) {
+    if (isString(manifest.version)) {
       // allow non-SemVer strings
-      dataC.version = data.version.trim()
+      manifestC.version = manifest.version.trim()
     }
     // endregion fix normalizations
 
     // work with a deep copy, because `normalizePackageData()` might modify the data
     const component = this.componentBuilder.makeComponent(
-      dataC as normalizePackageData.Package, type)
+      manifestC as normalizePackageData.Package, type)
     if (component === undefined) {
       this.console.debug('DEBUG | skip broken component: %j', locator)
       return undefined
@@ -296,7 +359,8 @@ export class BomBuilder {
   async * gatherDependencies (
     component: Component, pkg: Package,
     project: Project,
-    fetchManifest: ManifestFetcher
+    fetchManifest: ManifestFetcher,
+    fetchLicenseEvidences: LicenseEvidenceFetcher
   ): AsyncGenerator<Component> {
     // ATTENTION: multiple packages may have the same `identHash`, but the `locatorHash` is unique.
     const knownComponents = new Map<LocatorHash, Component>([[pkg.locatorHash, component]])
@@ -308,7 +372,8 @@ export class BomBuilder {
         let depComponent = knownComponents.get(depPkg.locatorHash)
         if (depComponent === undefined) {
           const _depIDN = structUtils.prettyLocatorNoColors(depPkg)
-          const _depC = await this.makeComponentFromPackage(depPkg, fetchManifest)
+          const _depC = await this.makeComponentFromPackage(depPkg,
+            fetchManifest, fetchLicenseEvidences)
           if (_depC === false) {
             // shall be skipped
             this.console.debug('DEBUG | skip impossible component %j', _depIDN)

src/commands.ts

@@ -112,6 +112,11 @@ export class MakeSbomCommand extends Command<CommandContext> {
         'This might result in loss of time- and random-based values.'
   })
 
+  gatherLicenseTexts = Option.Boolean('--gather-license-texts', false, {
+    description: 'Search for license files in components and include them as license evidence.\n' +
+        'This feature is experimental.'
+  })
+
   verbosity = Option.Counter('--verbose,-v', 1, {
     description: 'Increase the verbosity of messages.\n' +
         'Use multiple times to increase the verbosity even more.'
@@ -142,6 +147,7 @@ export class MakeSbomCommand extends Command<CommandContext> {
       mcType: this.mcType,
       shortPURLs: this.shortPURLs,
       outputReproducible: this.outputReproducible,
+      gatherLicenseTexts: this.gatherLicenseTexts,
       verbosity: this.verbosity,
       projectDir
     })
@@ -171,7 +177,8 @@ export class MakeSbomCommand extends Command<CommandContext> {
         omitDevDependencies: this.production,
         metaComponentType: this.mcType,
         reproducible: this.outputReproducible,
-        shortPURLs: this.shortPURLs
+        shortPURLs: this.shortPURLs,
+        gatherLicenseTexts: this.gatherLicenseTexts
       },
       myConsole
     )).buildFromWorkspace(workspace)

tests/README.md

@@ -15,14 +15,20 @@ Test files must follow the pattern `**.{spec,test}.[cm]?js`, to be picked up.
 Test runner is `mocha`, configured in [mocharc file](../.mocharc.js).
 
 ```shell
-npm test
+yarn run test
 ```
+
+To run specific tests only
+```shell
+yarn run test:node --grep "testname"
+```
+
 ### Snapshots
 
 Some tests check against snapshots.  
 To update these, set the env var `CYARN_TEST_UPDATE_SNAPSHOTS` to a non-falsy value.
 
 like so:
 ```shell
-CYARN_TEST_UPDATE_SNAPSHOTS=1 npm test
+CYARN_TEST_UPDATE_SNAPSHOTS=1 yarn run test
 ```