diff --git a/vulnerabilities/importers/__init__.py b/vulnerabilities/importers/__init__.py index 3f429f669..813179cb1 100644 --- a/vulnerabilities/importers/__init__.py +++ b/vulnerabilities/importers/__init__.py @@ -37,6 +37,7 @@ from vulnerabilities.pipelines import alpine_linux_importer from vulnerabilities.pipelines import github_importer from vulnerabilities.pipelines import gitlab_importer +from vulnerabilities.pipelines import misp_importer from vulnerabilities.pipelines import nginx_importer from vulnerabilities.pipelines import npm_importer from vulnerabilities.pipelines import nvd_importer @@ -78,6 +79,7 @@ nvd_importer.NVDImporterPipeline, pysec_importer.PyPIImporterPipeline, alpine_linux_importer.AlpineLinuxImporterPipeline, + misp_importer.MISPImporterPipeline, ] IMPORTERS_REGISTRY = { diff --git a/vulnerabilities/pipelines/misp_importer.py b/vulnerabilities/pipelines/misp_importer.py new file mode 100644 index 000000000..1c0e14418 --- /dev/null +++ b/vulnerabilities/pipelines/misp_importer.py @@ -0,0 +1,196 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +import json +import logging +import re +from datetime import timezone +from typing import Iterable + +import requests +from bs4 import BeautifulSoup +from dateutil import parser as dateparser +from packageurl import PackageURL +from univers.version_range import GenericVersionRange +from univers.version_range import VersionRange + +from vulnerabilities.importer import AdvisoryData +from vulnerabilities.importer import AffectedPackage +from vulnerabilities.importer import Reference +from vulnerabilities.importer import VulnerabilitySeverity +from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipeline +from vulnerabilities.severity_systems import CVSSV3 +from vulnerabilities.severity_systems import CVSSV31 +from vulnerabilities.utils import fetch_response +from vulnerabilities.utils import get_item + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +class MISPImporterPipeline(VulnerableCodeBaseImporterPipeline): + """Collect Advisories from MISP""" + + pipeline_id = "misp_importer" + spdx_license_expression = "CC BY-SA 3.0" + license_url = "https://www.misp-project.org/license/" + root_url = "https://www.misp-project.org/security/" + importer_name = "MISP Importer" + + def __init__(self): + super().__init__() + + @classmethod + def steps(cls): + return ( + cls.collect_and_store_advisories, + cls.import_new_advisories, + ) + + # num of advisories + def advisories_count(self) -> int: + return len(fetch_advisory_links(self.root_url)) + + # parse the response data + def collect_advisories(self) -> Iterable[AdvisoryData]: + advisory_links = fetch_advisory_links(self.root_url) + + for link in advisory_links: + advisory_data = fetch_advisory_data(link) + yield to_advisory_data(advisory_data) + + +def fetch_advisory_links(url): + """Fetches the advisory links listed on the URL,returns a list""" + r = fetch_response(url).content + soup = BeautifulSoup(r, "html.parser") + + h2 = soup.find(id="advisories") + # Find the
element containing the JSON data + pre_element = soup.find("pre", {"class": "json-container"}) + + # Extract the text content + json_text = pre_element.text + + # Parse the cleaned text as JSON + json_data = json.loads(json_text) + + # data + description = json_data["descriptions"][0]["value"] + cve_id = json_data["id"] + date_published = json_data["published"] + references = json_data["references"][0]["url"] + + # metrics + metrics = json_data["metrics"] + metrics_keys = list(metrics.keys()) + if "cvssMetricV31" in metrics_keys: + cve_score = { + "version": "cvssMetricV31", + "score": metrics["cvssMetricV31"][0]["cvssData"]["baseScore"], + } + else: + cve_score = { + "version": "cvssMetricV30", + "score": metrics["cvssMetricV30"][0]["cvssData"]["baseScore"], + } + + # affected version + match = re.search(r"\b\d+\.\d+\.\d+\b", description) + affected_version = match.group(0) + + return { + "description": description, + "alias": cve_id, + "date_published": date_published, + "references": references, + "cve_score": cve_score, + "affected_version": affected_version, + "url": url, + } + + +def to_advisory_data(raw_data) -> AdvisoryData: + """Parses extracted data to Advisory Data""" + # alias + alias = get_item(raw_data, "alias") + + # affected packages + affected_packages = [] + affected_version = get_item(raw_data, "affected_version") # list of list of affected versions + affected_packages.append( + AffectedPackage( + package=PackageURL(type="misp", name="MISP"), + affected_version_range=VersionRange.from_string(f"vers:generic/={affected_version}"), + ) + ) + + # score + if raw_data["cve_score"]["version"] == "cvssMetricV31": + severity = VulnerabilitySeverity( + system=CVSSV31, + value=raw_data["cve_score"]["score"], + scoring_elements="CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H", + ) + else: + severity = VulnerabilitySeverity( + system=CVSSV3, + value=raw_data["cve_score"]["score"], + scoring_elements="CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", + ) + + # Reference + references = [] + references.append( + Reference( + severities=[severity], + reference_id=alias, + url=get_item(raw_data, "references"), + ) + ) + + # description + description = get_item(raw_data, "description") + + # date published + date_published = get_item(raw_data, "date_published") + date_published = dateparser.parse(date_published, yearfirst=True).replace(tzinfo=timezone.utc) + + # url + url = get_item(raw_data, "url") + + return AdvisoryData( + aliases=alias, + summary=description, + affected_packages=affected_packages, + references=references, + url=url, + date_published=date_published, + ) diff --git a/vulnerabilities/tests/pipelines/test_misp_importer_pipeline.py b/vulnerabilities/tests/pipelines/test_misp_importer_pipeline.py new file mode 100644 index 000000000..d2f4ae43b --- /dev/null +++ b/vulnerabilities/tests/pipelines/test_misp_importer_pipeline.py @@ -0,0 +1,174 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import datetime +import json +from pathlib import Path +from unittest import mock +from unittest.mock import patch + +from packageurl import PackageURL +from univers.version_constraint import VersionConstraint +from univers.version_range import GenericVersionRange +from univers.versions import SemverVersion + +from vulnerabilities.importer import AdvisoryData +from vulnerabilities.importer import AffectedPackage +from vulnerabilities.importer import Reference +from vulnerabilities.importer import VulnerabilitySeverity +from vulnerabilities.pipelines import misp_importer +from vulnerabilities.severity_systems import Cvssv3ScoringSystem + +EXPECTED_DATA = Path(__file__).parent.parent / "test_data" / "misp" / "misp_expected.json" +HTML_DATA = Path(__file__).parent.parent / "test_data" / "misp" / "misp_test.html" +ADVISORY_HTML_DATA = Path(__file__).parent.parent / "test_data" / "misp" / "misp_advisory_test.html" + + +def load_expected_data(file): + with open(file) as f: + return json.load(f) + + +@mock.patch("requests.get") +def test_fetch_advisory_links(mock_get): + """Test fetching advisory links from MISP security advisories page.""" + + # Read mock HTML content + with open(HTML_DATA, "r", encoding="utf-8") as file: + mock_html_content = file.read() + + # Mock HTTP response + mock_response = mock.Mock() + mock_response.status_code = 200 + mock_response.content = mock_html_content.encode() + + mock_get.return_value = mock_response + + # Call function under test + links = misp_importer.fetch_advisory_links("https://www.misp-project.org/security/") + + # Ensure the links are extracted correctly + assert isinstance(links, list) + assert len(links) > 0 + assert "https://cve.circl.lu/vuln/fkie_cve-2015-5719" in links + + +@mock.patch("requests.get") +def test_fetch_advisory_data(mock_get): + """Test fetching and parsing advisory data from the CVE page.""" + + # Load expected advisory data from JSON file + expected_data = load_expected_data(EXPECTED_DATA) + + # Read mock HTML content + with open(ADVISORY_HTML_DATA, "r", encoding="utf-8") as file: + mock_html_content = file.read() + + # Create a proper mock response object + mock_response = mock.Mock() + mock_response.status_code = 200 + mock_response.text = mock_html_content + mock_response.content = mock_html_content.encode("utf-8") + + mock_get.return_value = mock_response + + # Call function under test + advisory_data = misp_importer.fetch_advisory_data( + "https://cve.circl.lu/vuln/fkie_cve-2015-5719" + ) + + # Validate extracted advisory data + assert advisory_data["alias"] == expected_data["aliases"] + assert advisory_data["description"] == expected_data["summary"] + assert advisory_data["date_published"] == expected_data["date_published"] + + +@mock.patch("requests.get") +def test_misp_importer_pipeline_collect_advisories(mock_get): + """Test the `collect_advisories` method in `MISPImporterPipeline`.""" + + with open(ADVISORY_HTML_DATA, "r", encoding="utf-8") as file: + mock_html_content = file.read() + + # Mock HTTP Response + mock_response = mock.Mock() + mock_response.status_code = 200 + mock_response.text = mock_html_content + mock_response.content = mock_html_content.encode("utf-8") + mock_get.return_value = mock_response + + # Initialize the pipeline + pipeline = misp_importer.MISPImporterPipeline() + + with mock.patch( + "vulnerabilities.pipelines.misp_importer.fetch_advisory_links" + ) as mock_links, mock.patch( + "vulnerabilities.pipelines.misp_importer.fetch_advisory_data" + ) as mock_data: + mock_links.return_value = ["https://cve.circl.lu/vuln/fkie_cve-2015-5719"] + mock_data.return_value = { + "description": "app/Controller/TemplatesController.php in Malware Information Sharing Platform (MISP) before 2.3.92 does not properly restrict filenames under the tmp/files/ directory, which has unspecified impact and attack vectors.", + "alias": "CVE-2015-5719", + "date_published": "2016-09-03T20:59:00.153", + "references": "http://www.securityfocus.com/bid/92740", + "cve_score": {"version": "cvssMetricV30", "score": 9.8}, + "affected_version": "2.3.92", + } + generator = pipeline.collect_advisories() + advisories = list(generator) + + assert len(advisories) == 1 + advisory = advisories[0] + + assert advisory.aliases == advisory_data.aliases + assert advisory.date_published == advisory_data.date_published + assert advisory.summary == advisory_data.summary + assert advisory.affected_packages == advisory.affected_packages + + +advisory_data = AdvisoryData( + aliases="CVE-2015-5719", + summary="app/Controller/TemplatesController.php in Malware Information Sharing Platform (MISP) before 2.3.92 does not properly restrict filenames under the tmp/files/ directory, which has unspecified impact and attack vectors.", + affected_packages=[ + AffectedPackage( + package=PackageURL( + type="misp", namespace=None, name="MISP", version=None, qualifiers={}, subpath=None + ), + affected_version_range=GenericVersionRange( + constraints=( + VersionConstraint(comparator="=", version=SemverVersion(string="2.3.92")), + ) + ), + fixed_version=None, + ) + ], + references=[ + Reference( + reference_id="CVE-2015-5719", + reference_type="", + url="http://www.securityfocus.com/bid/92740", + severities=[ + VulnerabilitySeverity( + system=Cvssv3ScoringSystem( + identifier="cvssv3", + name="CVSSv3 Base Score", + url="https://www.first.org/cvss/v3-0/", + notes="CVSSv3 base score and vector", + ), + value=9.8, + scoring_elements="CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", + published_at=None, + ) + ], + ) + ], + date_published=datetime.datetime(2016, 9, 3, 20, 59, 0, 153000, tzinfo=datetime.timezone.utc), + weaknesses=[], + url="https://cve.circl.lu/vuln/fkie_cve-2015-5719", +) diff --git a/vulnerabilities/tests/test_data/misp/misp_advisory_test.html b/vulnerabilities/tests/test_data/misp/misp_advisory_test.html new file mode 100644 index 000000000..246f37ee5 --- /dev/null +++ b/vulnerabilities/tests/test_data/misp/misp_advisory_test.html @@ -0,0 +1,140 @@ + +{ + "configurations": [ + { + "nodes": [ + { + "cpeMatch": [ + { + "criteria": "cpe:2.3:a:misp-project:malware_information_sharing_platform:*:*:*:*:*:*:*:*", + "matchCriteriaId": "FC0C651C-5962-47D4-AA2E-220BB4B1A1B2", + "versionEndIncluding": "2.3.91", + "vulnerable": true + } + ], + "negate": false, + "operator": "OR" + } + ] + } + ], + "cveTags": [], + "descriptions": [ + { + "lang": "en", + "value": "app/Controller/TemplatesController.php in Malware Information Sharing Platform (MISP) before 2.3.92 does not properly restrict filenames under the tmp/files/ directory, which has unspecified impact and attack vectors." + }, + { + "lang": "es", + "value": "app/Controller/TemplatesController.php en Malware Information Sharing Platform (MISP) en versiones anteriores a 2.3.92 no restringe adecuadamente nombres de archivo bajo el directorio tmp/files/, lo que tiene un impacto y vectores de ataque no especificados." + } + ], + "id": "CVE-2015-5719", + "lastModified": "2024-11-21T02:33:42.317", + "metrics": { + "cvssMetricV2": [ + { + "acInsufInfo": true, + "baseSeverity": "HIGH", + "cvssData": { + "accessComplexity": "LOW", + "accessVector": "NETWORK", + "authentication": "NONE", + "availabilityImpact": "COMPLETE", + "baseScore": 10.0, + "confidentialityImpact": "COMPLETE", + "integrityImpact": "COMPLETE", + "vectorString": "AV:N/AC:L/Au:N/C:C/I:C/A:C", + "version": "2.0" + }, + "exploitabilityScore": 10.0, + "impactScore": 10.0, + "obtainAllPrivilege": false, + "obtainOtherPrivilege": false, + "obtainUserPrivilege": false, + "source": "nvd@nist.gov", + "type": "Primary", + "userInteractionRequired": false + } + ], + "cvssMetricV30": [ + { + "cvssData": { + "attackComplexity": "LOW", + "attackVector": "NETWORK", + "availabilityImpact": "HIGH", + "baseScore": 9.8, + "baseSeverity": "CRITICAL", + "confidentialityImpact": "HIGH", + "integrityImpact": "HIGH", + "privilegesRequired": "NONE", + "scope": "UNCHANGED", + "userInteraction": "NONE", + "vectorString": "CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", + "version": "3.0" + }, + "exploitabilityScore": 3.9, + "impactScore": 5.9, + "source": "nvd@nist.gov", + "type": "Primary" + } + ] + }, + "published": "2016-09-03T20:59:00.153", + "references": [ + { + "source": "cve@mitre.org", + "url": "http://www.securityfocus.com/bid/92740" + }, + { + "source": "cve@mitre.org", + "tags": [ + "Issue Tracking", + "Patch" + ], + "url": "https://github.com/MISP/MISP/commit/27cc167c3355ec76292235d7f5f4e0016bfd7699" + }, + { + "source": "cve@mitre.org", + "tags": [ + "Third Party Advisory" + ], + "url": "https://www.circl.lu/advisory/CVE-2015-5719/" + }, + { + "source": "af854a3a-2127-422b-91ae-364da2661108", + "url": "http://www.securityfocus.com/bid/92740" + }, + { + "source": "af854a3a-2127-422b-91ae-364da2661108", + "tags": [ + "Issue Tracking", + "Patch" + ], + "url": "https://github.com/MISP/MISP/commit/27cc167c3355ec76292235d7f5f4e0016bfd7699" + }, + { + "source": "af854a3a-2127-422b-91ae-364da2661108", + "tags": [ + "Third Party Advisory" + ], + "url": "https://www.circl.lu/advisory/CVE-2015-5719/" + } + ], + "sourceIdentifier": "cve@mitre.org", + "vulnStatus": "Modified", + "weaknesses": [ + { + "description": [ + { + "lang": "en", + "value": "NVD-CWE-noinfo" + } + ], + "source": "nvd@nist.gov", + "type": "Primary" + } + ] + }+ + \ No newline at end of file diff --git a/vulnerabilities/tests/test_data/misp/misp_expected.json b/vulnerabilities/tests/test_data/misp/misp_expected.json new file mode 100644 index 000000000..4b5e46818 --- /dev/null +++ b/vulnerabilities/tests/test_data/misp/misp_expected.json @@ -0,0 +1,35 @@ +{ + "aliases": "CVE-2015-5719", + "summary": "app/Controller/TemplatesController.php in Malware Information Sharing Platform (MISP) before 2.3.92 does not properly restrict filenames under the tmp/files/ directory, which has unspecified impact and attack vectors.", + "affected_packages": [ + { + "package": { + "type": "misp", + "namespace": "", + "name": "MISP", + "version": "", + "qualifiers": "", + "subpath": "" + }, + "affected_version_range": "vers:generic/2.3.92", + "fixed_version": null + } + ], + "references": [ + { + "reference_id": "CVE-2015-5719", + "reference_type": "", + "url": "http://www.securityfocus.com/bid/92740", + "severities": [ + { + "system": "cvssv3", + "value": 9.8, + "scoring_elements": "CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H" + } + ] + } + ], + "date_published": "2016-09-03T20:59:00.153", + "weaknesses": [], + "url": "https://cve.circl.lu/vuln/fkie_cve-2015-5719" +} \ No newline at end of file diff --git a/vulnerabilities/tests/test_data/misp/misp_test.html b/vulnerabilities/tests/test_data/misp/misp_test.html new file mode 100644 index 000000000..c2627e81f --- /dev/null +++ b/vulnerabilities/tests/test_data/misp/misp_test.html @@ -0,0 +1,100 @@ + +Advisories
+
add_misp_export
in app/Controller/EventsController.php does not properly check for a valid file upload.__uploadLogo
in app/Controller/OrganisationsController.php does not properly check for a valid logo upload.