Merge branch 'main' into improve-ssvc-yaml-display

Author: Mahaboobunnisa123

CHANGELOG.rst

@@ -13,10 +13,16 @@ Version v37.0.0
 - We have added new models AdvisoryV2, AdvisoryAlias, AdvisoryReference, AdvisorySeverity, AdvisoryWeakness, PackageV2 and CodeFixV2.
 - We are using ``avid`` as an internal advisory ID for uniquely identifying advisories.
 - We have a new route ``/v2`` which only support package search which has information on packages that are reported to be affected or fixing by advisories.
-- This version introduces ``/api/v2/advisories-packages`` which has information on packages that are reported to be affected or fixing by advisories.
+- This version introduces ``/api/v3/packages`` which has information on packages that are reported to be affected or fixing by advisories.
 - Pipeline Dashboard improvements #1920.
 - Throttle API requests based on user permissions #1909.
 - Add pipeline to compute Advisory ToDos #1764
+- Use related advisory severity to calculate exploitibility, weighted severity and risk scores
+- Migrate all importers to use the new advisory models. All new advisories have a unique AVID and all importers will use this AVID as the unique identifier for advisories instead of CVE ID or other identifiers used by the data sources #1881.
+- Handle advisories with same and related data https://github.com/aboutcode-org/vulnerablecode/issues/2099.
+- Add a pipeline for exporting VulnerableCode data to FederatedCode #2110.
+- Plan storing of exploits and EPSS based advisories #2069.
+
 
 Version v36.1.3
 ---------------------

PIPELINES-AVID.rst

@@ -0,0 +1,74 @@
+.. list-table:: Pipeline AVID Mapping
+   :header-rows: 1
+   :widths: 35 65
+
+   * - pipeline name
+     - AVID
+   * - alpine_linux_importer_v2
+     - {package_name}/{distroversion}/{version}/{vulnerability_id}
+   * - aosp_dataset_fix_commits
+     - CVE ID of the record
+   * - apache_httpd_importer_v2
+     - CVE ID of the record
+   * - apache_kafka_importer_v2
+     - CVE ID of the record
+   * - apache_tomcat_importer_v2
+     - {page_id}/{cve_id}
+   * - archlinux_importer_v2
+     - AVG ID of the record
+   * - curl_importer_v2
+     - CURL-CVE ID of the record
+   * - debian_importer_v2
+     - {package_name}/{debian_record_id}
+   * - elixir_security_importer_v2
+     - {package_name}/{file_id}
+   * - epss_importer_v2
+     - CVE ID of the record
+   * - fireeye_importer_v2
+     - {file_id}
+   * - gentoo_importer_v2
+     - GLSA ID of the record
+   * - github_osv_importer_v2
+     - ID of the OSV record
+   * - gitlab_importer_v2
+     - Identifier of the GitLab community advisory record
+   * - istio_importer_v2
+     - ISTIO-SECURITY-<ID>
+   * - mattermost_importer_v2
+     - MMSA-<ID>
+   * - mozilla_importer_v2
+     - MFSA-<ID>
+   * - nginx_importer_v2
+     - First alias of the record
+   * - nodejs_security_wg
+     - NPM-<ID>
+   * - nvd_importer_v2
+     - CVE ID of the record
+   * - openssl_importer_v2
+     - CVE ID of the record
+   * - oss_fuzz_importer_v2
+     - ID of the OSV record
+   * - postgresql_importer_v2
+     - CVE ID of the record
+   * - project-kb-msr-2019_v2
+     - Vulnerability ID of the record
+   * - project-kb-statements_v2
+     - Vulnerability ID of the record
+   * - pypa_importer_v2
+     - ID of the OSV record
+   * - pysec_importer_v2
+     - ID of the OSV record
+   * - redhat_importer_v2
+     - RHSA ID of the record
+   * - retiredotnet_importer_v2
+     - retiredotnet-{file_id}
+   * - ruby_importer_v2
+     - {file_id}
+   * - suse_importer_v2
+     - CVE ID of the record
+   * - ubuntu_osv_importer_v2
+     - ID of the OSV record
+   * - vulnrichment_importer_v2
+     - CVE ID of the record
+   * - xen_importer_v2
+     - XSA-<ID>

vulnerabilities/pipelines/enhance_with_exploitdb.py

@@ -78,19 +78,16 @@ def add_exploit(self):
 
 
 def add_vulnerability_exploit(row, logger):
-    vulnerabilities = set()
-
     aliases = row["codes"].split(";") if row["codes"] else []
 
     if not aliases:
         return 0
 
-    for raw_alias in aliases:
-        try:
-            if alias := Alias.objects.get(alias=raw_alias):
-                vulnerabilities.add(alias.vulnerability)
-        except Alias.DoesNotExist:
-            continue
+    vulnerabilities = (
+        Alias.objects.filter(alias__in=aliases, vulnerability__isnull=False)
+        .values_list("vulnerability_id", flat=True)
+        .distinct()
+    )
 
     if not vulnerabilities:
         logger(f"No vulnerability found for aliases {aliases}")
@@ -104,7 +101,7 @@ def add_vulnerability_exploit(row, logger):
         add_exploit_references(row["codes"], row["source_url"], row["file"], vulnerability, logger)
         try:
             Exploit.objects.update_or_create(
-                vulnerability=vulnerability,
+                vulnerability_id=vulnerability,
                 data_source="Exploit-DB",
                 defaults={
                     "date_added": date_added,
@@ -125,7 +122,7 @@ def add_vulnerability_exploit(row, logger):
     return 1
 
 
-def add_exploit_references(ref_id, direct_url, path, vul, logger):
+def add_exploit_references(ref_id, direct_url, path, vul_id, logger):
     url_map = {
         "file_url": f"https://gitlab.com/exploit-database/exploitdb/-/blob/main/{path}",
         "direct_url": direct_url,
@@ -144,7 +141,7 @@ def add_exploit_references(ref_id, direct_url, path, vul, logger):
 
                 if created:
                     VulnerabilityRelatedReference.objects.get_or_create(
-                        vulnerability=vul,
+                        vulnerability_id=vul_id,
                         reference=ref,
                     )
 

vulnerabilities/pipelines/enhance_with_kev.py

@@ -71,26 +71,29 @@ def add_vulnerability_exploit(kev_vul, logger):
     if not cve_id:
         return 0
 
-    vulnerability = None
-    try:
-        if alias := Alias.objects.get(alias=cve_id):
-            vulnerability = alias.vulnerability
-    except Alias.DoesNotExist:
+    vulnerabilities = (
+        Alias.objects.filter(alias=cve_id, vulnerability__isnull=False)
+        .values_list("vulnerability", flat=True)
+        .distinct()
+    )
+
+    if not vulnerabilities:
         logger(f"No vulnerability found for aliases {cve_id}")
         return 0
 
-    Exploit.objects.update_or_create(
-        vulnerability=vulnerability,
-        data_source="KEV",
-        defaults={
-            "description": kev_vul["shortDescription"],
-            "date_added": kev_vul["dateAdded"],
-            "required_action": kev_vul["requiredAction"],
-            "due_date": kev_vul["dueDate"],
-            "notes": kev_vul["notes"],
-            "known_ransomware_campaign_use": True
-            if kev_vul["knownRansomwareCampaignUse"] == "Known"
-            else False,
-        },
-    )
+    for vulnerability in vulnerabilities:
+        Exploit.objects.update_or_create(
+            vulnerability_id=vulnerability,
+            data_source="KEV",
+            defaults={
+                "description": kev_vul["shortDescription"],
+                "date_added": kev_vul["dateAdded"],
+                "required_action": kev_vul["requiredAction"],
+                "due_date": kev_vul["dueDate"],
+                "notes": kev_vul["notes"],
+                "known_ransomware_campaign_use": True
+                if kev_vul["knownRansomwareCampaignUse"] == "Known"
+                else False,
+            },
+        )
     return 1

vulnerabilities/pipelines/enhance_with_metasploit.py

@@ -66,7 +66,6 @@ def add_vulnerability_exploits(self):
 
 
 def add_vulnerability_exploit(record, logger):
-    vulnerabilities = set()
     references = record.get("references", [])
 
     interesting_references = [
@@ -76,12 +75,11 @@ def add_vulnerability_exploit(record, logger):
     if not interesting_references:
         return 0
 
-    for ref in interesting_references:
-        try:
-            if alias := Alias.objects.get(alias=ref):
-                vulnerabilities.add(alias.vulnerability)
-        except Alias.DoesNotExist:
-            continue
+    vulnerabilities = (
+        Alias.objects.filter(alias__in=interesting_references, vulnerability__isnull=False)
+        .values_list("vulnerability", flat=True)
+        .distinct()
+    )
 
     if not vulnerabilities:
         logger(f"No vulnerability found for aliases {interesting_references}")
@@ -107,7 +105,7 @@ def add_vulnerability_exploit(record, logger):
 
     for vulnerability in vulnerabilities:
         Exploit.objects.update_or_create(
-            vulnerability=vulnerability,
+            vulnerability_id=vulnerability,
             data_source="Metasploit",
             defaults={
                 "description": description,

vulnerabilities/pipelines/v2_importers/alpine_linux_importer.py

@@ -7,6 +7,7 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
+import json
 import logging
 from pathlib import Path
 from typing import Any
@@ -244,4 +245,5 @@ def load_advisories(
                 references=references,
                 affected_packages=affected_packages,
                 url=url,
+                original_advisory_text=json.dumps(pkg_infos, indent=2, ensure_ascii=False),
             )

vulnerabilities/pipelines/v2_importers/apache_tomcat_importer.py

@@ -111,6 +111,7 @@ def collect_advisories(self) -> Iterable[AdvisoryDataV2]:
                         summary=advisory_list[0].summary,
                         affected_packages=affected_packages,
                         url=page_url,
+                        original_advisory_text=str(content),
                     )
 
             except Exception as e:

vulnerabilities/pipelines/v2_importers/archlinux_importer.py

@@ -14,10 +14,13 @@
 from packageurl import PackageURL
 from univers.version_range import ArchLinuxVersionRange
 
+from vulnerabilities import severity_systems
 from vulnerabilities.importer import AdvisoryDataV2
 from vulnerabilities.importer import AffectedPackageV2
 from vulnerabilities.importer import ReferenceV2
+from vulnerabilities.importer import VulnerabilitySeverity
 from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2
+from vulnerabilities.severity_systems import SCORING_SYSTEMS
 from vulnerabilities.utils import fetch_response
 
 
@@ -53,7 +56,9 @@ def collect_advisories(self) -> Iterable[AdvisoryDataV2]:
     def parse_advisory(self, record) -> AdvisoryDataV2:
         affected_packages = []
         references = []
+        severities = []
         avg_name = record.get("name")
+        severity = record.get("severity")
         aliases = record.get("issues", [])
         aliases.extend(record.get("advisories", []))
         summary = record.get("type", "")
@@ -92,13 +97,26 @@ def parse_advisory(self, record) -> AdvisoryDataV2:
                 )
             )
 
+        if severity not in severity_systems.ARCHLINUX.choices:
+            self.log(f"Unknown severity {severity} for {avg_name}")
+            severity = None
+        if severity:
+            severities = [
+                VulnerabilitySeverity(
+                    system=severity_systems.ARCHLINUX,
+                    value=severity,
+                    url="https://security.archlinux.org/{avg_name}.json",
+                )
+            ]
+
         return AdvisoryDataV2(
             advisory_id=avg_name,
             aliases=aliases,
             summary=summary,
             references=references,
             affected_packages=affected_packages,
+            severities=severities,
             weaknesses=[],
             url=f"https://security.archlinux.org/{avg_name}.json",
-            original_advisory_text=json.dumps(record),
+            original_advisory_text=json.dumps(record, indent=2, ensure_ascii=False),
         )

vulnerabilities/pipelines/v2_importers/debian_importer.py

@@ -7,6 +7,7 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
+import json
 import re
 from typing import Any
 from typing import Iterable
@@ -171,6 +172,7 @@ def parse(self, pkg_name: str, records: Mapping[str, Any]) -> Iterable[AdvisoryD
                 references=references,
                 weaknesses=weaknesses,
                 url=f"https://security-tracker.debian.org/tracker/{record_identifier}",
+                original_advisory_text=json.dumps(record, indent=2, ensure_ascii=False),
             )
 
 

vulnerabilities/pipelines/v2_importers/epss_importer_v2.py

@@ -83,4 +83,5 @@ def collect_advisories(self) -> Iterable[AdvisoryDataV2]:
                 severities=[severity],
                 references=[references],
                 url=self.advisory_url,
+                original_advisory_text=",".join(epss_row),
             )