diff --git a/vulnerabilities/importers/__init__.py b/vulnerabilities/importers/__init__.py index 3f429f669..813179cb1 100644 --- a/vulnerabilities/importers/__init__.py +++ b/vulnerabilities/importers/__init__.py @@ -37,6 +37,7 @@ from vulnerabilities.pipelines import alpine_linux_importer from vulnerabilities.pipelines import github_importer from vulnerabilities.pipelines import gitlab_importer +from vulnerabilities.pipelines import misp_importer from vulnerabilities.pipelines import nginx_importer from vulnerabilities.pipelines import npm_importer from vulnerabilities.pipelines import nvd_importer @@ -78,6 +79,7 @@ nvd_importer.NVDImporterPipeline, pysec_importer.PyPIImporterPipeline, alpine_linux_importer.AlpineLinuxImporterPipeline, + misp_importer.MISPImporterPipeline, ] IMPORTERS_REGISTRY = { diff --git a/vulnerabilities/pipelines/misp_importer.py b/vulnerabilities/pipelines/misp_importer.py new file mode 100644 index 000000000..1c0e14418 --- /dev/null +++ b/vulnerabilities/pipelines/misp_importer.py @@ -0,0 +1,196 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +import json +import logging +import re +from datetime import timezone +from typing import Iterable + +import requests +from bs4 import BeautifulSoup +from dateutil import parser as dateparser +from packageurl import PackageURL +from univers.version_range import GenericVersionRange +from univers.version_range import VersionRange + +from vulnerabilities.importer import AdvisoryData +from vulnerabilities.importer import AffectedPackage +from vulnerabilities.importer import Reference +from vulnerabilities.importer import VulnerabilitySeverity +from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipeline +from vulnerabilities.severity_systems import CVSSV3 +from vulnerabilities.severity_systems import CVSSV31 +from vulnerabilities.utils import fetch_response +from vulnerabilities.utils import get_item + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +class MISPImporterPipeline(VulnerableCodeBaseImporterPipeline): + """Collect Advisories from MISP""" + + pipeline_id = "misp_importer" + spdx_license_expression = "CC BY-SA 3.0" + license_url = "https://www.misp-project.org/license/" + root_url = "https://www.misp-project.org/security/" + importer_name = "MISP Importer" + + def __init__(self): + super().__init__() + + @classmethod + def steps(cls): + return ( + cls.collect_and_store_advisories, + cls.import_new_advisories, + ) + + # num of advisories + def advisories_count(self) -> int: + return len(fetch_advisory_links(self.root_url)) + + # parse the response data + def collect_advisories(self) -> Iterable[AdvisoryData]: + advisory_links = fetch_advisory_links(self.root_url) + + for link in advisory_links: + advisory_data = fetch_advisory_data(link) + yield to_advisory_data(advisory_data) + + +def fetch_advisory_links(url): + """Fetches the advisory links listed on the URL,returns a list""" + r = fetch_response(url).content + soup = BeautifulSoup(r, "html.parser") + + h2 = soup.find(id="advisories") + # Find the